def build_config_space(experiment):
'''
Build ray config space from flattened spec.search
Specify a config space in spec using `"{key}__{space_type}": {v}`.
Where `{space_type}` is `grid_search` of `ray.tune`, or any function name of `np.random`:
- `grid_search`: str/int/float. v = list of choices
- `choice`: str/int/float. v = list of choices
- `randint`: int. v = [low, high)
- `uniform`: float. v = [low, high)
- `normal`: float. v = [mean, stdev)
For example:
- `"explore_anneal_epi__randint": [10, 60],` will sample integers uniformly from 10 to 60 for `explore_anneal_epi`,
- `"lr__uniform": [0.001, 0.1]`, and it will sample `lr` using `np.random.uniform(0.001, 0.1)`
If any key uses `grid_search`, it will be combined exhaustively in combination with other random sampling.
'''
config_space = {}
for k, v in util.flatten_dict(experiment.spec['search']).items():
if '__' in k:
key, space_type = k.split('__')
else:
key, space_type = k, 'grid_search'
if space_type == 'grid_search':
config_space[key] = grid_search(v)
elif space_type == 'choice':
config_space[key] = lambda spec, v=v: random.choice(v)
else:
np_fn = getattr(np.random, space_type)
config_space[key] = lambda spec, v=v: np_fn(*v)
return config_space
def testConvertDragonfly(self):
from ray.tune.suggest.dragonfly import DragonflySearch
# Grid search not supported, should raise ValueError
with self.assertRaises(ValueError):
DragonflySearch.convert_search_space(
{"grid": tune.grid_search([0, 1])})
config = {
"a": tune.sample.Categorical([2, 3, 4]).uniform(),
"b": {
"x": tune.sample.Integer(0, 5).quantized(2),
"y": 4,
"z": tune.sample.Float(1e-4, 1e-2).loguniform()
}
}
with self.assertRaises(ValueError):
converted_config = DragonflySearch.convert_search_space(config)
config = {
"a": 4,
"b": {
"z": tune.sample.Float(1e-4, 1e-2).loguniform()
}
}
dragonfly_config = [{
"name": "b/z",
"type": "float",
"min": 1e-4,
"max": 1e-2
}]
converted_config = DragonflySearch.convert_search_space(config)
np.random.seed(1234)
searcher1 = DragonflySearch(optimizer="bandit",
domain="euclidean",
space=converted_config,
metric="none",
mode="max")
config1 = searcher1.suggest("0")
np.random.seed(1234)
searcher2 = DragonflySearch(optimizer="bandit",
domain="euclidean",
space=dragonfly_config,
metric="none",
mode="max")
config2 = searcher2.suggest("0")
self.assertEqual(config1, config2)
self.assertLess(config2["b"]["z"], 1e-2)
searcher = DragonflySearch()
invalid_config = {"a/b": tune.uniform(4.0, 8.0)}
with self.assertRaises(ValueError):
searcher.set_search_properties("none", "max", invalid_config)
invalid_config = {"a": {"b/c": tune.uniform(4.0, 8.0)}}
with self.assertRaises(ValueError):
searcher.set_search_properties("none", "max", invalid_config)
searcher = DragonflySearch(optimizer="bandit",
domain="euclidean",
metric="a",
mode="max")
analysis = tune.run(_mock_objective,
config=config,
search_alg=searcher,
num_samples=1)
trial = analysis.trials[0]
self.assertLess(trial.config["b"]["z"], 1e-2)
mixed_config = {
"a": tune.uniform(5, 6),
"b": tune.uniform(8, 9) # Cannot mix List and Dict
}
searcher = DragonflySearch(space=mixed_config,
optimizer="bandit",
domain="euclidean",
metric="a",
mode="max")
config = searcher.suggest("0")
self.assertTrue(5 <= config["a"] <= 6)
self.assertTrue(8 <= config["b"] <= 9)
def testConvertBayesOpt(self):
from ray.tune.suggest.bayesopt import BayesOptSearch
# Grid search not supported, should raise ValueError
with self.assertRaises(ValueError):
BayesOptSearch.convert_search_space(
{"grid": tune.grid_search([0, 1])})
config = {
"a": tune.sample.Categorical([2, 3, 4]).uniform(),
"b": {
"x": tune.sample.Integer(0, 5).quantized(2),
"y": 4,
"z": tune.sample.Float(1e-4, 1e-2).loguniform()
}
}
with self.assertRaises(ValueError):
converted_config = BayesOptSearch.convert_search_space(config)
config = {"b": {"z": tune.sample.Float(1e-4, 1e-2).loguniform()}}
bayesopt_config = {"b/z": (1e-4, 1e-2)}
converted_config = BayesOptSearch.convert_search_space(config)
searcher1 = BayesOptSearch(space=converted_config,
metric="none",
mode="max")
searcher2 = BayesOptSearch(space=bayesopt_config,
metric="none",
mode="max")
config1 = searcher1.suggest("0")
config2 = searcher2.suggest("0")
self.assertEqual(config1, config2)
self.assertLess(1e-4, config1["b"]["z"])
self.assertLess(config1["b"]["z"], 1e-2)
searcher = BayesOptSearch()
invalid_config = {"a/b": tune.uniform(4.0, 8.0)}
with self.assertRaises(ValueError):
searcher.set_search_properties("none", "max", invalid_config)
invalid_config = {"a": {"b/c": tune.uniform(4.0, 8.0)}}
with self.assertRaises(ValueError):
searcher.set_search_properties("none", "max", invalid_config)
searcher = BayesOptSearch(metric="b", mode="max")
analysis = tune.run(_mock_objective,
config=config,
search_alg=searcher,
num_samples=1)
trial = analysis.trials[0]
self.assertLess(trial.config["b"]["z"], 1e-2)
mixed_config = {"a": tune.uniform(5, 6), "b": (8., 9.)}
searcher = BayesOptSearch(space=mixed_config, metric="a", mode="max")
config = searcher.suggest("0")
self.assertTrue(5 <= config["a"] <= 6)
self.assertTrue(8 <= config["b"] <= 9)
def testPointsToEvaluateBasicVariantAdvanced(self):
config = {
"grid_1": tune.grid_search(["a", "b", "c", "d"]),
"grid_2": tune.grid_search(["x", "y", "z"]),
"nested": {
"random":
tune.uniform(2., 10.),
"dependent":
tune.sample_from(lambda spec: -1. * spec.config.nested.random)
}
}
points = [
{
"grid_1": "b"
},
{
"grid_2": "z"
},
{
"grid_1": "a",
"grid_2": "y"
},
{
"nested": {
"random": 8.0
}
},
]
from ray.tune.suggest.basic_variant import BasicVariantGenerator
# grid_1 * grid_2 are 3 * 4 = 12 variants per complete grid search
# However if one grid var is set by preset variables, that run
# is excluded from grid search.
# Point 1 overwrites grid_1, so the first trial only grid searches
# over grid_2 (3 trials).
# The remaining 5 trials search over the whole space (5 * 12 trials)
searcher = BasicVariantGenerator(points_to_evaluate=[points[0]])
exp = Experiment(run=_mock_objective,
name="test",
config=config,
num_samples=6)
searcher.add_configurations(exp)
self.assertEqual(searcher.total_samples, 1 * 3 + 5 * 12)
# Point 2 overwrites grid_2, so the first trial only grid searches
# over grid_1 (4 trials).
# The remaining 5 trials search over the whole space (5 * 12 trials)
searcher = BasicVariantGenerator(points_to_evaluate=[points[1]])
exp = Experiment(run=_mock_objective,
name="test",
config=config,
num_samples=6)
searcher.add_configurations(exp)
self.assertEqual(searcher.total_samples, 1 * 4 + 5 * 12)
# Point 3 overwrites grid_1 and grid_2, so the first trial does not
# grid search.
# The remaining 5 trials search over the whole space (5 * 12 trials)
searcher = BasicVariantGenerator(points_to_evaluate=[points[2]])
exp = Experiment(run=_mock_objective,
name="test",
config=config,
num_samples=6)
searcher.add_configurations(exp)
self.assertEqual(searcher.total_samples, 1 + 5 * 12)
# When initialized with all points, the first three trials are
# defined by the logic above. Only 3 trials are grid searched
# compeletely.
searcher = BasicVariantGenerator(points_to_evaluate=points)
exp = Experiment(run=_mock_objective,
name="test",
config=config,
num_samples=6)
searcher.add_configurations(exp)
self.assertEqual(searcher.total_samples, 1 * 3 + 1 * 4 + 1 + 3 * 12)
# Run this and confirm results
analysis = tune.run(exp, search_alg=searcher)
configs = [trial.config for trial in analysis.trials]
self.assertEqual(len(configs), searcher.total_samples)
self.assertTrue(all(config["grid_1"] == "b"
for config in configs[0:3]))
self.assertTrue(all(config["grid_2"] == "z"
for config in configs[3:7]))
self.assertTrue(configs[7]["grid_1"] == "a"
and configs[7]["grid_2"] == "y")
self.assertTrue(configs[8]["nested"]["random"] == 8.0)
self.assertTrue(configs[8]["nested"]["dependent"] == -8.0)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--smoke-test",
action="store_true",
help="Finish quickly for testing")
parser.add_argument(
"--server-address",
type=str,
default=None,
required=False,
help="The address of server to connect to if using "
"Ray Client.",
)
args, _ = parser.parse_known_args()
if args.server_address and not args.smoke_test:
import ray
ray.init(f"ray://{args.server_address}")
analysis = tune.run(
MNISTTrainable,
metric="test_loss",
mode="min",
stop={"training_iteration": 5 if args.smoke_test else 50},
verbose=1,
config={"hiddens": tune.grid_search([32, 64, 128])},
)
print("Best hyperparameters found were: ", analysis.best_config)
reward_path=discrim_path,
),
)
assert run.status == "COMPLETED"
_check_rollout_stats(run.result)
PARALLEL_CONFIG_UPDATES = [
dict(
sacred_ex_name="expert_demos",
base_named_configs=["cartpole", "fast"],
n_seeds=2,
search_space={
"config_updates": {
"init_rl_kwargs": {
"learning_rate": tune.grid_search([3e-4, 1e-4])
},
}
},
),
dict(
sacred_ex_name="train_adversarial",
base_named_configs=["cartpole", "gail", "fast"],
base_config_updates={
# Need absolute path because raylet runs in different working directory.
"rollout_path":
osp.abspath(
"tests/data/expert_models/cartpole_0/rollouts/final.pkl"),
},
search_space={
"config_updates": {
tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)
# !!! Example of using the ray.tune Python API !!!
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument(
'--smoke-test', action='store_true', help='Finish quickly for testing')
args, _ = parser.parse_known_args()
register_trainable('train_mnist', train)
mnist_spec = {
'run': 'train_mnist',
'stop': {
'mean_accuracy': 0.99,
'time_total_s': 600,
},
'config': {
'activation': grid_search(['relu', 'elu', 'tanh']),
# You can pass any serializable object as well
'foo': grid_search([np.array([1, 2]),
np.array([2, 3])]),
},
}
if args.smoke_test:
mnist_spec['stop']['training_iteration'] = 2
ray.init()
run_experiments({'tune_mnist_test': mnist_spec})
def step(self, action):
assert action in [0, 1], action
if action == 0 and self.cur_pos > 0:
self.cur_pos -= 1
elif action == 1:
self.cur_pos += 1
done = self.cur_pos >= self.end_pos
return [self.cur_pos], 1 if done else 0, done, {}
if __name__ == "__main__":
# Can also register the env creator function explicitly with:
# register_env("corridor", lambda config: SimpleCorridor(config))
ray.init()
run_experiments({
"demo": {
"run": "PPO",
"env": SimpleCorridor, # or "corridor" if registered above
"stop": {
"timesteps_total": 10000,
},
"config": {
"lr": grid_search([1e-2, 1e-4, 1e-6]), # try different lrs
"num_workers": 1, # parallelism
"env_config": {
"corridor_length": 5,
},
},
},
})
def get_config():
return {
# === Environment ===
"env":
"Navigation",
"env_config":
tune.grid_search([
{
"deceleration_zones": None
},
{
"deceleration_zones": {
"center": [[0.0, 0.0]],
"decay": [2.0]
}
},
]),
# === Replay Buffer ===
"buffer_size":
int(1e4),
# === Optimization ===
# Name of Pytorch optimizer class for paremetrized policy
"torch_optimizer":
"Adam",
# Keyword arguments to be passed to the on-policy optimizer
"torch_optimizer_options": {
"model": {
"lr": 3e-4
},
"value": {
"lr": 3e-4
},
"policy": {
"lr": 3e-4
},
},
# Clip gradient norms by this value
"max_grad_norm":
1e3,
# === Regularization ===
"kl_schedule": {
"initial_coeff": 0.2,
"desired_kl": 0.01,
"adaptation_coeff": 1.01,
"threshold": 1.0,
},
# === Network ===
# Size and activation of the fully connected networks computing the logits
# for the policy, value function and model. No layers means the component is
# linear in states and/or actions.
"module": {
"actor": {
"encoder": {
"units": (64, 64),
"activation": "ReLU",
"initializer_options": {
"name": "xavier_uniform"
},
},
"input_dependent_scale": False,
},
"critic": {
"target_vf": True,
"encoder": {
"units": (64, 64),
"activation": "ReLU",
"initializer_options": {
"name": "xavier_uniform"
},
},
},
"model": {
"residual": True,
"input_dependent_scale": False,
"encoder": {
"units": (64, 64),
"activation": "ReLU",
"delay_action": True,
"initializer_options": {
"name": "xavier_uniform"
},
},
},
},
# === RolloutWorker ===
"rollout_fragment_length":
1,
"batch_mode":
"complete_episodes",
# === Trainer ===
"train_batch_size":
32,
"timesteps_per_iteration":
200,
# === Exploration ===
"exploration_config": {
"pure_exploration_steps": 200
},
# === Evaluation ===
"evaluation_interval":
5,
"evaluation_num_episodes":
5,
}
parser.add_argument("--ray-address", type=str, default=None)
parser.add_argument("--fixedweight", action="store_true")
parser.add_argument("--resume", type=str, default=None)
args = parser.parse_args()
ray.init(redis_address=args.ray_address)
exp_name = "L0-MNIST-{}-{}".format(time.strftime("%Y%m%d-%H%M%S"),
uuid.uuid1())
print("Running experiment {}".format(exp_name))
analysis = tune.run(StochasticMNISTExperiment,
name=exp_name,
num_samples=args.samples,
config={
"lr": args.lr,
"l0_strength": tune.grid_search(args.l0),
"l2_strength": tune.grid_search(args.l2),
"model_type": args.model,
"learn_weight": not args.fixedweight,
},
stop={"training_iteration": args.epochs},
checkpoint_at_end=True,
resources_per_trial={
"cpu": 1,
"gpu": (1 if torch.cuda.is_available() else 0)
},
loggers=(JsonLogger, CSVLogger, TFLoggerPlus),
verbose=1)
print(("To browse results, instantiate "
'`tune.Analysis("~/ray_results/{}")`').format(exp_name))
from __future__ import division
from __future__ import print_function
import ray
from ray.tune import grid_search, run_experiments
from env import CarlaEnv, ENV_CONFIG
from models import register_carla_model
from scenarios import TOWN2_STRAIGHT
env_config = ENV_CONFIG.copy()
env_config.update({
"verbose": False,
"x_res": 80,
"y_res": 80,
"squash_action_logits": grid_search([False, True]),
"use_depth_camera": False,
"discrete_actions": False,
"server_map": "/Game/Maps/Town02",
"reward_function": grid_search(["custom", "corl2017"]),
"scenarios": TOWN2_STRAIGHT,
})
register_carla_model()
redis_address = ray.services.get_node_ip_address() + ":6379"
ray.init(redis_address=redis_address)
run_experiments({
"carla-a3c": {
"run": "A3C",
"env": CarlaEnv,
def to_grid_search(self):
"""Returns a ray.tune.grid_search structure.
Apply sampling to get discrete choices.
"""
return grid_search(self.choices)
def to_grid_search(self):
"""Returns a ray.tune.grid_search structure.
Contains all the choices inside and can be expanded by ray.tune.
"""
return grid_search(self.choices)
def testConvertNevergrad(self):
from ray.tune.suggest.nevergrad import NevergradSearch
import nevergrad as ng
# Grid search not supported, should raise ValueError
with self.assertRaises(ValueError):
NevergradSearch.convert_search_space(
{"grid": tune.grid_search([0, 1])})
config = {
"a": tune.sample.Categorical([2, 3, 4]).uniform(),
"b": {
"x": tune.sample.Integer(0, 5).quantized(2),
"y": 4,
"z": tune.sample.Float(1e-4, 1e-2).loguniform()
}
}
converted_config = NevergradSearch.convert_search_space(config)
nevergrad_config = ng.p.Dict(
a=ng.p.Choice([2, 3, 4]),
b=ng.p.Dict(x=ng.p.Scalar(lower=0, upper=5).set_integer_casting(),
z=ng.p.Log(lower=1e-4, upper=1e-2)))
searcher1 = NevergradSearch(optimizer=ng.optimizers.OnePlusOne,
space=converted_config,
metric="a",
mode="max")
searcher2 = NevergradSearch(optimizer=ng.optimizers.OnePlusOne,
space=nevergrad_config,
metric="a",
mode="max")
np.random.seed(1234)
config1 = searcher1.suggest("0")
np.random.seed(1234)
config2 = searcher2.suggest("0")
self.assertEqual(config1, config2)
self.assertIn(config1["a"], [2, 3, 4])
self.assertIn(config1["b"]["x"], list(range(5)))
self.assertLess(1e-4, config1["b"]["z"])
self.assertLess(config1["b"]["z"], 1e-2)
searcher = NevergradSearch(optimizer=ng.optimizers.OnePlusOne,
metric="a",
mode="max")
analysis = tune.run(_mock_objective,
config=config,
search_alg=searcher,
num_samples=1)
trial = analysis.trials[0]
assert trial.config["a"] in [2, 3, 4]
mixed_config = {
"a": tune.uniform(5, 6),
"b": tune.uniform(8, 9) # Cannot mix Nevergrad cfg and tune
}
searcher = NevergradSearch(space=mixed_config,
optimizer=ng.optimizers.OnePlusOne,
metric="a",
mode="max")
config = searcher.suggest("0")
self.assertTrue(5 <= config["a"] <= 6)
self.assertTrue(8 <= config["b"] <= 9)
def main():
args = parser.parse_args()
config = generate_config(args)
# env = CityFlowEnvRay(config)
# eng = cityflow.Engine(config["cityflow_config_file"], thread_num = config["thread_num"])
# config["eng"] = [eng,]
# print(config["eng"])
num_agents = len(config["intersection_id"])
grouping = {"group_1": [id_ for id_ in sorted(config["intersection_id"])]}
obs_space = Tuple(
[CityFlowEnvRay.observation_space for _ in range(num_agents)])
act_space = Tuple([CityFlowEnvRay.action_space for _ in range(num_agents)])
register_env(
"cityflow_multi", lambda config_: CityFlowEnvRay(config_).
with_agent_groups(grouping, obs_space=obs_space, act_space=act_space))
if args.algo == "QMIX":
config_ = {
# "num_workers": 2,
"num_gpus_per_worker": 0,
"sample_batch_size": 4,
"num_cpus_per_worker": 30,
"train_batch_size": 3,
"exploration_final_eps": 0.0,
"num_workers": 1,
"mixer": grid_search(["vdn"]), # "qmix"
"double_q": True,
"exploration_fraction": 0.1,
"exploration_final_eps": 0.2,
"target_network_update_freq": 500,
"buffer_size": 10000,
"learning_starts": 500,
"lr": 0.0005,
"env_config": config
}
group = True
elif args.algo == "APEX_QMIX":
config_ = {
"num_gpus": 1,
"num_workers": 2,
"optimizer": {
"num_replay_buffer_shards": 1,
},
"min_iter_time_s": 3,
"buffer_size": 2000,
"learning_starts": 300,
"train_batch_size": 64,
"sample_batch_size": 32,
"target_network_update_freq": 100,
"timesteps_per_iteration": 1000,
"env_config": config
}
group = True
else:
config_ = {}
group = False
ray.init()
if not args.rollout:
tune.run(
args.algo,
stop={"timesteps_total": args.epoch * args.num_step},
checkpoint_freq=args.save_freq,
config=dict(config_, **{"env": "cityflow_multi"}),
)
else:
print("#####################")
print("rollout...")
print("#####################")
tune.run(
args.algo,
stop={
"timesteps_total": 3000,
"training_iteration": 0
},
restore=args.ckpt,
resume=False,
config=dict(config_, **{"env": "cityflow_multi"}),
)
def testConvertZOOpt(self):
from ray.tune.suggest.zoopt import ZOOptSearch
from zoopt import ValueType
# Grid search not supported, should raise ValueError
with self.assertRaises(ValueError):
ZOOptSearch.convert_search_space(
{"grid": tune.grid_search([0, 1])})
config = {
"a": tune.sample.Categorical([2, 3, 4]).uniform(),
"b": {
"x": tune.sample.Integer(0, 5).quantized(2),
"y": tune.sample.Categorical([2, 4, 6, 8]).uniform(),
"z": tune.sample.Float(1e-4, 1e-2).loguniform()
}
}
# Does not support categorical variables
with self.assertRaises(ValueError):
converted_config = ZOOptSearch.convert_search_space(config)
config = {
"a": 2,
"b": {
"x": tune.sample.Integer(0, 5).uniform(),
"y": tune.sample.Categorical([2, 4, 6, 8]).uniform(),
"z": tune.sample.Float(-3, 7).uniform().quantized(1e-4)
}
}
converted_config = ZOOptSearch.convert_search_space(config)
zoopt_config = {
"b/x": (ValueType.DISCRETE, [0, 5], True),
"b/y": (ValueType.GRID, [2, 4, 6, 8]),
"b/z": (ValueType.CONTINUOUS, [-3, 7], 1e-4),
}
zoopt_search_config = {"parallel_num": 4}
searcher1 = ZOOptSearch(dim_dict=converted_config,
budget=5,
metric="a",
mode="max",
**zoopt_search_config)
searcher2 = ZOOptSearch(dim_dict=zoopt_config,
budget=5,
metric="a",
mode="max",
**zoopt_search_config)
np.random.seed(1234)
config1 = searcher1.suggest("0")
np.random.seed(1234)
config2 = searcher2.suggest("0")
self.assertEqual(config1, config2)
self.assertIn(config1["b"]["x"], list(range(5)))
self.assertIn(config1["b"]["y"], [2, 4, 6, 8])
self.assertLess(-3, config1["b"]["z"])
self.assertLess(config1["b"]["z"], 7)
searcher = ZOOptSearch(budget=5,
metric="a",
mode="max",
**zoopt_search_config)
analysis = tune.run(_mock_objective,
config=config,
search_alg=searcher,
num_samples=1)
trial = analysis.trials[0]
self.assertIn(trial.config["b"]["y"], [2, 4, 6, 8])
mixed_config = {
"a": tune.uniform(5, 6),
"b": (ValueType.CONTINUOUS, [8, 9], 1e-4)
}
searcher = ZOOptSearch(dim_dict=mixed_config,
budget=5,
metric="a",
mode="max",
**zoopt_search_config)
config = searcher.suggest("0")
self.assertTrue(5 <= config["a"] <= 6)
self.assertTrue(8 <= config["b"] <= 9)
register_trainable("train_cifar10", Cifar10Model)
train_spec = {
"run": "train_cifar10",
"trial_resources": {
"cpu": 1,
"gpu": 1
},
"stop": {
"mean_accuracy": 0.80,
"training_iteration": 30,
},
"config": {
"epochs": 1,
"batch_size": 64,
"lr": grid_search([10**-4, 10**-5]),
"decay": lambda spec: spec.config.lr / 100.0,
"dropout": grid_search([0.25, 0.5]),
},
"repeat": 4,
}
if args.smoke_test:
train_spec["config"]["lr"] = 10**-4
train_spec["config"]["dropout"] = 0.5
ray.init()
pbt = PopulationBasedTraining(time_attr="training_iteration",
reward_attr="mean_accuracy",
perturbation_interval=10,
act_space = Tuple([
TwoStepGame.action_space,
TwoStepGame.action_space,
])
register_env(
"grouped_twostep",
lambda config: TwoStepGame(config).with_agent_groups(
grouping, obs_space=obs_space, act_space=act_space))
if args.run == "QMIX":
config = {
"sample_batch_size": 4,
"train_batch_size": 32,
"exploration_final_eps": 0.0,
"num_workers": 0,
"mixer": grid_search([None, "qmix", "vdn"]),
}
group = True
elif args.run == "APEX_QMIX":
config = {
"num_gpus": 0,
"num_workers": 2,
"optimizer": {
"num_replay_buffer_shards": 1,
},
"min_iter_time_s": 3,
"buffer_size": 1000,
"learning_starts": 1000,
"train_batch_size": 128,
"sample_batch_size": 32,
"target_network_update_freq": 500,
help="Finish quickly for testing")
args, _ = parser.parse_known_args()
train_spec = {
"run": SNN_model,
"trial_resources": {
"cpu": 1
},
"stop": {
"test_accuracy": 0.99,
"training_iteration": 3000,
},
"config": {
"T": 200,
"dfa": 0,
"hl": grid_search([1, 2]),
"dropr": 0.0,
"evt": 0,
"lim": 1.0,
"norm": 0.0,
"rel": 0,
"delt": 5,
"dr": 1,
"init": 0,
"clp": 1,
"hThr": grid_search([0.2, 0.4]),
"outputThr": 1.0,
"batch_size": 10,
"fr": 1.0,
"bias": 0.0,
"twin": 1,
tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)
# !!! Example of using the ray.tune Python API !!!
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--smoke-test',
action='store_true',
help='Finish quickly for testing')
args, _ = parser.parse_known_args()
register_trainable('train_mnist', train)
mnist_spec = {
'stop': {
'mean_accuracy': 0.99,
'time_total_s': 600,
},
'config': {
'activation': grid_search(['relu', 'elu', 'tanh']),
# You can pass any serializable object as well
'foo': grid_search([np.array([1, 2]),
np.array([2, 3])]),
},
}
if args.smoke_test:
mnist_spec['stop']['training_iteration'] = 2
ray.init()
tune.run('train_mnist', name='tune_mnist_test', **mnist_spec)
def ax_optimize_accuracy_weightsparsity(exploring_trainable,
followup_trainable, experiment_name,
script_dir, alphas, parameters,
num_training_iterations,
samples_per_frontier_config):
"""
Optimize a Ray Trainable's "mean_accuracy" and "inference_nz", using Ax to
select hyperparameters. This procedure will pick up any existing results for
this experiment and plug them into the Ax model, then will generate new
results. To optimize two objectives, this procedure chooses a set of
projections from two scalar objectives to a single scalar objective. These
projections are defined by alpha*log(error) + (1 - alpha)*log(inference_nz).
The procedure loops over the different alphas and rebuilds the Ax model and
generates new results for each. All results go into the same Ray experiment
folder, so each search (i.e. each alpha) benefits from all of the other
searches. Finally, this procedure finds all configurations at the Pareto
frontier of the two objectives and gathers additional results for each until
every configuration at the frontier is well-sampled.
"""
experiment_dir = os.path.expanduser(f"~/ray_results/{experiment_name}")
parser = argparse.ArgumentParser()
parser.add_argument("--run-initial-configs", action="store_true")
parser.add_argument("--num-iterations", type=int, default=1)
parser.add_argument("--num-random", type=int, default=16)
parser.add_argument("--num-configs", type=int, default=16)
args = parser.parse_args()
ray.init()
if args.run_initial_configs:
input_dir = Path(script_dir) / "best_configs"
input_file = input_dir / f"{experiment_name}.json"
if input_file.exists():
print("STAGE 0: Test initial suggested hyperparameters")
with open(input_file, "r") as f:
frontier_trials = json.load(f)
configs = [dict(config) for config, _, _ in frontier_trials]
# Only include parameters specified in the parameters list.
configs = [{
parameter["name"]: config[parameter["name"]]
for parameter in parameters
} for config in configs]
tune.run(
exploring_trainable,
name=experiment_name,
config=tune.grid_search(configs),
num_samples=1,
resources_per_trial={
"cpu": 1,
"gpu": (1 if torch.cuda.is_available() else 0)
},
loggers=DEFAULT_LOGGERS,
verbose=1,
)
num_random_remaining = args.num_random
for _ in range(args.num_iterations):
print("STAGE 1: Hyperparameter search")
for alpha in alphas:
if num_random_remaining + args.num_configs == 0:
break
known_trials = get_ray_trials(experiment_dir, parameters)
known_trials = [
(config, result) for config, result in known_trials
if result["training_iteration"] == num_training_iterations
]
ax_client = ax_client_with_explicit_strategy(
num_random_remaining, args.num_configs)
ax_client.create_experiment(parameters=parameters,
objective_name="hyperparameter_loss",
minimize=True)
for config, result in known_trials:
loss = hyperparameter_loss(config, result, alpha)
_, trial_index = ax_client.attach_trial(dict(config))
ax_client.complete_trial(
trial_index=trial_index,
raw_data={"hyperparameter_loss": (loss, None)})
tune.run(
insert_hyperparameter_loss(exploring_trainable),
name=experiment_name,
search_alg=AxSearch(
ax_client,
max_concurrent=(torch.cuda.device_count()
if torch.cuda.is_available() else 1)),
config=dict(alpha=alpha),
num_samples=args.num_configs + num_random_remaining,
resources_per_trial={
"cpu": 1,
"gpu": (1 if torch.cuda.is_available() else 0)
},
loggers=DEFAULT_LOGGERS,
verbose=1,
)
num_random_remaining = 0
# Verify the frontier
print(
"STAGE 2: Make sure we have enough samples for each point at the "
"frontier")
while True:
resample_configs = []
for config, results in get_frontier_trials(
experiment_dir, parameters, num_training_iterations):
if len(results) < samples_per_frontier_config:
resample_configs += (samples_per_frontier_config -
len(results)) * [dict(config)]
if len(resample_configs) > 0:
tune.run(
followup_trainable,
name=experiment_name,
config=tune.grid_search(resample_configs),
num_samples=1,
resources_per_trial={
"cpu": 1,
"gpu": (1 if torch.cuda.is_available() else 0)
},
loggers=DEFAULT_LOGGERS,
verbose=1,
)
else:
break
output = []
for config, results in get_frontier_trials(experiment_dir, parameters,
num_training_iterations):
acc = np.mean([result["mean_accuracy"] for result in results])
nz = np.mean([result["inference_nz"] for result in results])
output.append((config, acc, nz))
output_dir = Path(script_dir) / "output"
os.makedirs(output_dir, exist_ok=True)
output_file = output_dir / f"{experiment_name}.json"
with open(output_file, "w") as f:
print(f"Saving {output_file}")
json.dump(output, f)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("--test", action="store_true")
parser.add_argument("--num-gpus", type=int, default=8)
parser.add_argument("--num-seeds", type=int, default=3)
parser.add_argument("--num-agents", type=int, default=10)
parser.add_argument("--exp-name", type=str, default="")
parser.add_argument("--walker", action="store_true")
args = parser.parse_args()
if not args.test:
assert args.exp_name
humanoid_config = {
# can change
"update_steps": tune.grid_search([200000, 500000, 1000000]),
"env": MultiAgentEnvWrapper,
"env_config": {
"env_name": "Humanoid-v2",
"num_agents": args.num_agents
},
# should be fixed
"clip_param": 0.2,
"kl_coeff": 1.0,
"num_sgd_iter": 20,
"lr": 0.0003,
"gamma": 0.995,
"lambda": 0.95,
'sgd_minibatch_size': 4096,
'train_batch_size': 65536,
from ray import tune
def objective(step, alpha, beta):
return (0.1 + alpha * step / 100)**(-1) + beta * 0.1
def training_function(config):
# Hyperparameters
alpha, beta = config["alpha"], config["beta"]
for step in range(10):
# Iterative training function - can be any arbitrary training procedure.
intermediate_score = objective(step, alpha, beta)
# Feed the score back back to Tune.
tune.report(mean_loss=intermediate_score)
analysis = tune.run(training_function,
config={
"alpha": tune.grid_search([0.001, 0.01, 0.1]),
"beta": tune.choice([1, 2, 3])
})
print("Best config: ", analysis.get_best_config(metric="mean_loss",
mode="min"))
# Get a dataframe for analyzing trial results.
print('arbitrary')
df = analysis.results_df
def main():
# for the user defined module in code_dir, need to be imported in functions
# sys.path.insert(0, code_dir)
# import parameters
# import basic_src.io_function as io_function
# import workflow.whole_procedure as whole_procedure
# from utility.eva_report_to_tables import read_accuracy_multi_reports
loc_dir = "./ray_results"
# tune_name = "tune_traning_para_tesia"
# tune_name = "tune_backbone_para_tesia"
tune_name = "tune_backbone_largeBatchS_tesia"
file_folders = io_function.get_file_list_by_pattern(
os.path.join(loc_dir, tune_name), '*')
# if len(file_folders) > 1:
# b_resume = True
# else:
# b_resume = False
# try to resume after when through all (some failed), they always complain:
# "Trials did not complete", incomplete_trials, so don't resume.
b_resume = False
# max_failures = 2,
# stop = tune.function(stop_function),
analysis = tune.run(
training_function,
resources_per_trial={
"gpu": 2
}, # use two GPUs, 12 CPUs on tesia # "cpu": 14, don't limit cpu, eval.py will not use all
local_dir=loc_dir,
name=tune_name,
# fail_fast=True, # Stopping after the first failure
log_to_file=("stdout.log",
"stderr.log"), #Redirecting stdout and stderr to files
trial_name_creator=tune.function(trial_name_string),
trial_dirname_creator=tune.function(trial_dir_string),
resume=b_resume,
config={
"lr":
tune.grid_search([0.007, 0.014,
0.28]), # ,0.007, 0.014, 0.028,0.056
"iter_num":
tune.grid_search([30000]), # , 60000,90000,
"batch_size":
tune.grid_search([48, 64, 96]), # 8,16,32 16, 32, 64, 128
"backbone":
tune.grid_search(backbones),
"buffer_size":
tune.grid_search([300]), # 600
"training_data_per":
tune.grid_search([0.9]), #, 0.8
"data_augmentation":
tune.grid_search(['blur,crop,bright,contrast,noise']),
'data_aug_ignore_classes':
tune.grid_search(['class_0'])
}
# config={
# "lr": tune.grid_search([0.014]), # ,0.007, 0.014, 0.028,0.056
# "iter_num": tune.grid_search([30000]), # , 60000,90000
# "batch_size": tune.grid_search([8]), # 16, 32, 64, 128
# "backbone": tune.grid_search(backbones),
# "buffer_size": tune.grid_search([300]),
# "training_data_per": tune.grid_search([0.9]),
# "data_augmentation": tune.grid_search(['scale, bright, contrast, noise']),
# 'data_aug_ignore_classes':tune.grid_search(['class_0',''])
# }
)
print("Best config: ",
analysis.get_best_config(metric="overall_miou", mode="max"))
# Get a dataframe for analyzing trial results.
df = analysis.results_df
output_file = 'training_miou_ray_tune_%s.xlsx' % (
datetime.now().strftime("%Y%m%d_%H%M%S"))
with pd.ExcelWriter(output_file) as writer:
df.to_excel(writer) # , sheet_name='accuracy table'
# set format
# workbook = writer.book
# format = workbook.add_format({'num_format': '#0.000'})
# acc_talbe_sheet = writer.sheets['accuracy table']
# acc_talbe_sheet.set_column('G:I',None,format)
print('write trial results to %s' % output_file)
train_loss=loss.item(),
best_test_acc=100. * test_correct /
test_num) # report metrics
tune.register_trainable("train", train)
all_trials = tune.run_experiments({
"awesome": {
"run": "train",
"repeat": 1,
# "trial_resources": {
# "cpu": 8,
# "gpu": 1,
# },
"stop": {
"best_test_acc": 90,
},
#"stop": {"epoch": 1},
"config": {
"lr": tune.grid_search(list(uniform.rvs(0, size=3))),
"momentum": tune.grid_search(list(uniform.rvs(0, size=1))),
},
"local_dir": "ray_results",
"max_failures": 1
}
})
ray.error_info()
ray.global_state.log_files()
def testConvertAx(self):
from ray.tune.suggest.ax import AxSearch
from ax.service.ax_client import AxClient
# Grid search not supported, should raise ValueError
with self.assertRaises(ValueError):
AxSearch.convert_search_space({"grid": tune.grid_search([0, 1])})
config = {
"a": tune.sample.Categorical([2, 3, 4]).uniform(),
"b": {
"x": tune.sample.Integer(0, 5).quantized(2),
"y": 4,
"z": tune.sample.Float(1e-4, 1e-2).loguniform()
}
}
converted_config = AxSearch.convert_search_space(config)
ax_config = [
{
"name": "a",
"type": "choice",
"values": [2, 3, 4]
},
{
"name": "b/x",
"type": "range",
"bounds": [0, 5],
"value_type": "int"
},
{
"name": "b/y",
"type": "fixed",
"value": 4
},
{
"name": "b/z",
"type": "range",
"bounds": [1e-4, 1e-2],
"value_type": "float",
"log_scale": True
},
]
client1 = AxClient(random_seed=1234)
client1.create_experiment(parameters=converted_config,
objective_name="a",
minimize=False)
searcher1 = AxSearch(ax_client=client1)
client2 = AxClient(random_seed=1234)
client2.create_experiment(parameters=ax_config,
objective_name="a",
minimize=False)
searcher2 = AxSearch(ax_client=client2)
config1 = searcher1.suggest("0")
config2 = searcher2.suggest("0")
self.assertEqual(config1, config2)
self.assertIn(config1["a"], [2, 3, 4])
self.assertIn(config1["b"]["x"], list(range(5)))
self.assertEqual(config1["b"]["y"], 4)
self.assertLess(1e-4, config1["b"]["z"])
self.assertLess(config1["b"]["z"], 1e-2)
searcher = AxSearch(metric="a", mode="max")
analysis = tune.run(_mock_objective,
config=config,
search_alg=searcher,
num_samples=1)
trial = analysis.trials[0]
assert trial.config["a"] in [2, 3, 4]
mixed_config = {"a": tune.uniform(5, 6), "b": tune.uniform(8, 9)}
searcher = AxSearch(space=mixed_config, metric="a", mode="max")
config = searcher.suggest("0")
self.assertTrue(5 <= config["a"] <= 6)
self.assertTrue(8 <= config["b"] <= 9)
def testCheckpointDownload(self):
analysis = tune.run(
train,
config={"train_id": tune.grid_search([0, 1, 2, 3])},
local_dir=self.local_experiment_dir,
verbose=2)
# Inject the sync config (this is usually done by `tune.run()`)
analysis._sync_config = tune.SyncConfig(upload_dir=self.cloud_target)
# Pretend we have all checkpoints on cloud storage (durable)
shutil.rmtree(self.fake_cloud_dir, ignore_errors=True)
shutil.copytree(self.local_experiment_dir, self.fake_cloud_dir)
# Pretend we don't have two trials on local storage
shutil.rmtree(analysis.trials[1].logdir)
shutil.rmtree(analysis.trials[2].logdir)
shutil.rmtree(analysis.trials[3].logdir)
cp0 = analysis.get_best_checkpoint(analysis.trials[0], "score", "max")
cp1 = analysis.get_best_checkpoint(analysis.trials[1], "score", "max")
cp2 = analysis.get_best_checkpoint(analysis.trials[2], "score", "max")
cp3 = analysis.get_best_checkpoint(analysis.trials[3], "score", "max")
def _load_cp(cd):
with open(os.path.join(cd, "checkpoint.json"), "rt") as f:
return json.load(f)
with patch("ray.tune.cloud._clear_bucket", self._clear_bucket), \
patch("ray.tune.cloud._download_from_bucket",
self._fake_download_from_bucket), \
patch("ray.tune.cloud._upload_to_bucket",
self._fake_upload_to_bucket):
#######
# Case: Checkpoint exists on local dir. Copy to other local dir.
other_local_dir = tempfile.mkdtemp()
cp0.save(other_local_dir)
self.assertTrue(os.path.exists(cp0.local_path))
cp_content = _load_cp(other_local_dir)
self.assertEquals(cp_content["train_id"], 0)
self.assertEquals(cp_content["score"], 9)
cp_content_2 = _load_cp(cp0.local_path)
self.assertEquals(cp_content, cp_content_2)
# Clean up
shutil.rmtree(other_local_dir)
#######
# Case: Checkpoint does not exist on local dir, download from cloud
# store in experiment dir.
# Directory is empty / does not exist before
self.assertFalse(os.path.exists(cp1.local_path))
# Save!
cp1.save()
# Directory is not empty anymore
self.assertTrue(os.listdir(cp1.local_path))
cp_content = _load_cp(cp1.local_path)
self.assertEquals(cp_content["train_id"], 1)
self.assertEquals(cp_content["score"], 9)
#######
# Case: Checkpoint does not exist on local dir, download from cloud
# store into other local dir.
# Directory is empty / does not exist before
self.assertFalse(os.path.exists(cp2.local_path))
other_local_dir = tempfile.mkdtemp()
# Save!
cp2.save(other_local_dir)
# Directory still does not exist (as we save to other dir)
self.assertFalse(os.path.exists(cp2.local_path))
cp_content = _load_cp(other_local_dir)
self.assertEquals(cp_content["train_id"], 2)
self.assertEquals(cp_content["score"], 9)
# Clean up
shutil.rmtree(other_local_dir)
#######
# Case: Checkpoint does not exist on local dir, download from cloud
# and store onto other cloud.
# Local dir does not exist
self.assertFalse(os.path.exists(cp3.local_path))
# First cloud exists
self.assertTrue(os.listdir(self.fake_cloud_dir))
# Second cloud does not exist
self.assertFalse(os.listdir(self.second_fake_cloud_dir))
# Trigger save
cp3.save(self.second_cloud_target)
# Local dir now exists
self.assertTrue(os.path.exists(cp3.local_path))
# First cloud exists
self.assertTrue(os.listdir(self.fake_cloud_dir))
# Second cloud now exists!
self.assertTrue(os.listdir(self.second_fake_cloud_dir))
cp_content = _load_cp(self.second_fake_cloud_dir)
self.assertEquals(cp_content["train_id"], 3)
self.assertEquals(cp_content["score"], 9)
def testConvertBOHB(self):
from ray.tune.suggest.bohb import TuneBOHB
import ConfigSpace
# Grid search not supported, should raise ValueError
with self.assertRaises(ValueError):
TuneBOHB.convert_search_space({"grid": tune.grid_search([0, 1])})
config = {
"a": tune.sample.Categorical([2, 3, 4]).uniform(),
"b": {
"x": tune.sample.Integer(0, 5).quantized(2),
"y": 4,
"z": tune.sample.Float(1e-4, 1e-2).loguniform()
}
}
converted_config = TuneBOHB.convert_search_space(config)
bohb_config = ConfigSpace.ConfigurationSpace()
bohb_config.add_hyperparameters([
ConfigSpace.CategoricalHyperparameter("a", [2, 3, 4]),
ConfigSpace.UniformIntegerHyperparameter("b/x",
lower=0,
upper=4,
q=2),
ConfigSpace.UniformFloatHyperparameter("b/z",
lower=1e-4,
upper=1e-2,
log=True)
])
converted_config.seed(1234)
bohb_config.seed(1234)
searcher1 = TuneBOHB(space=converted_config, metric="a", mode="max")
searcher2 = TuneBOHB(space=bohb_config, metric="a", mode="max")
config1 = searcher1.suggest("0")
config2 = searcher2.suggest("0")
self.assertEqual(config1, config2)
self.assertIn(config1["a"], [2, 3, 4])
self.assertIn(config1["b"]["x"], list(range(5)))
self.assertLess(1e-4, config1["b"]["z"])
self.assertLess(config1["b"]["z"], 1e-2)
searcher = TuneBOHB(metric="a", mode="max")
analysis = tune.run(_mock_objective,
config=config,
search_alg=searcher,
num_samples=1)
trial = analysis.trials[0]
self.assertIn(trial.config["a"], [2, 3, 4])
self.assertEqual(trial.config["b"]["y"], 4)
mixed_config = {
"a": tune.uniform(5, 6),
"b": tune.uniform(8, 9) # Cannot mix ConfigSpace and Dict
}
searcher = TuneBOHB(space=mixed_config, metric="a", mode="max")
config = searcher.suggest("0")
self.assertTrue(5 <= config["a"] <= 6)
self.assertTrue(8 <= config["b"] <= 9)
'n_initial_exploration_steps': int(1e3),
'n_epochs': 200,
}
},
'DDL': {
'type': 'DDL',
'kwargs': {
'reparameterize': REPARAMETERIZE,
'lr': 3e-4,
'target_update_interval': 1,
'tau': 5e-3,
'target_entropy': 'auto',
'action_prior': 'uniform',
'n_initial_exploration_steps': int(1e3),
'n_epochs': 200,
'train_distance_fn_every_n_steps': tune.grid_search([16, 64]),
'ext_reward_coeff': tune.grid_search([0.25, 0.5]),
'normalize_ext_reward_gamma': tune.grid_search([1]),
'use_env_intrinsic_reward': tune.grid_search([True]),
'ddl_symmetric': tune.grid_search([False]),
# 'ddl_clip_length': tune.grid_search([None, 20, 50]),
'ddl_train_steps': tune.grid_search([2, 4, 10]),
'ddl_batch_size': 256,
#'rnd_int_rew_coeff': tune.grid_search([None, 1]),
},
'rnd_params': {
'convnet_params': {
'conv_filters': (16, 32, 64),
'conv_kernel_sizes': (3, 3, 3),
'conv_strides': (2, 2, 2),
def testConvertHEBO(self):
from ray.tune.suggest.hebo import HEBOSearch
from hebo.design_space.design_space import DesignSpace
import torch
# Grid search not supported, should raise ValueError
with self.assertRaises(ValueError):
HEBOSearch.convert_search_space({"grid": tune.grid_search([0, 1])})
config = {
"a": tune.sample.Categorical([2, 3, 4]).uniform(),
"b": {
"x": tune.sample.Integer(0, 5),
"y": 4,
"z": tune.sample.Float(1e-4, 1e-2).loguniform()
}
}
converted_config = HEBOSearch.convert_search_space(config)
hebo_space_config = [
{
"name": "a",
"type": "cat",
"categories": [2, 3, 4]
},
{
"name": "b/x",
"type": "int",
"lb": 0,
"ub": 5
},
{
"name": "b/z",
"type": "pow",
"lb": 1e-4,
"ub": 1e-2
},
]
hebo_space = DesignSpace().parse(hebo_space_config)
searcher1 = HEBOSearch(space=converted_config,
metric="a",
mode="max",
random_state_seed=123)
searcher2 = HEBOSearch(space=hebo_space,
metric="a",
mode="max",
random_state_seed=123)
np.random.seed(1234)
torch.manual_seed(1234)
config1 = searcher1.suggest("0")
np.random.seed(1234)
torch.manual_seed(1234)
config2 = searcher2.suggest("0")
self.assertEqual(config1, config2)
self.assertIn(config1["a"], [2, 3, 4])
self.assertIn(config1["b"]["x"], list(range(5)))
self.assertLess(1e-4, config1["b"]["z"])
self.assertLess(config1["b"]["z"], 1e-2)
searcher = HEBOSearch(metric="a", mode="max", random_state_seed=123)
analysis = tune.run(_mock_objective,
config=config,
search_alg=searcher,
num_samples=1)
trial = analysis.trials[0]
self.assertIn(trial.config["a"], [2, 3, 4])
self.assertEqual(trial.config["b"]["y"], 4)
def get_variant_spec_base(universe, domain, task, task_eval, policy, algorithm,
from_vision):
algorithm_params = ALGORITHM_PARAMS_BASE
algorithm_params = deep_update(
algorithm_params, ALGORITHM_PARAMS_ADDITIONAL.get(algorithm, {}))
variant_spec = {
'git_sha':
get_git_rev(),
'environment_params': {
'training': {
'domain':
domain,
'task':
task,
'universe':
universe,
'kwargs':
get_environment_params(universe, domain, task, from_vision),
},
'evaluation': {
'domain':
domain,
'task':
task_eval,
'universe':
universe,
'kwargs':
(tune.sample_from(lambda spec: (spec.get('config', spec)[
'environment_params']['training'].get('kwargs')))
if task == task_eval else get_environment_params(
universe, domain, task_eval, from_vision)),
},
},
'policy_params':
deep_update(POLICY_PARAMS_BASE[policy],
POLICY_PARAMS_FOR_DOMAIN[policy].get(domain, {})),
'exploration_policy_params': {
'type': 'UniformPolicy',
'kwargs': {
'observation_keys':
tune.sample_from(lambda spec: (spec.get('config', spec)[
'policy_params']['kwargs'].get('observation_keys')))
},
},
'Q_params': {
'type': 'double_feedforward_Q_function',
'kwargs': {
'hidden_layer_sizes': (M, M),
'observation_keys':
tune.sample_from(lambda spec: (spec.get('config', spec)[
'policy_params']['kwargs'].get('observation_keys'))),
'observation_preprocessors_params': {}
}
},
'distance_fn_params':
get_distance_fn_params(universe, domain, task),
'algorithm_params':
algorithm_params,
'replay_pool_params': {
'type': 'SimpleReplayPool',
'kwargs': {
# 'max_size': int(5e5),
'max_size': tune.grid_search([int(5e4)]),
}
},
'sampler_params': {
'type': 'SimpleSampler',
'kwargs': {
'max_path_length': get_max_path_length(universe, domain, task),
'min_pool_size': 50,
'batch_size': 256, # tune.grid_search([128, 256]),
'store_last_n_paths': 20,
}
},
'run_params': {
'seed': tune.sample_from(lambda spec: np.random.randint(0, 10000)),
'checkpoint_at_end': False,
'checkpoint_frequency': tune.sample_from(get_checkpoint_frequency),
'checkpoint_replay_pool': False,
},
}
# Filter out parts of the state relating to the object when training from pixels
env_kwargs = variant_spec['environment_params']['training']['kwargs']
if from_vision and "device_path" not in env_kwargs.keys():
env_obs_keys = env_kwargs.get('observation_keys', tuple())
non_image_obs_keys = tuple(key for key in env_obs_keys
if key != 'pixels')
variant_spec['replay_pool_params']['kwargs'][
'obs_save_keys'] = non_image_obs_keys
non_object_obs_keys = tuple(key for key in env_obs_keys
if 'object' not in key)
variant_spec['policy_params']['kwargs'][
'observation_keys'] = variant_spec['exploration_policy_params'][
'kwargs']['observation_keys'] = variant_spec['Q_params'][
'kwargs']['observation_keys'] = variant_spec[
'distance_fn_params']['kwargs'][
'observation_keys'] = non_object_obs_keys
return variant_spec
def testConvertOptuna(self):
from ray.tune.suggest.optuna import OptunaSearch, param
from optuna.samplers import RandomSampler
# Grid search not supported, should raise ValueError
with self.assertRaises(ValueError):
OptunaSearch.convert_search_space(
{"grid": tune.grid_search([0, 1])})
config = {
"a": tune.sample.Categorical([2, 3, 4]).uniform(),
"b": {
"x": tune.sample.Integer(0, 5).quantized(2),
"y": 4,
"z": tune.sample.Float(1e-4, 1e-2).loguniform()
}
}
converted_config = OptunaSearch.convert_search_space(config)
optuna_config = [
param.suggest_categorical("a", [2, 3, 4]),
param.suggest_int("b/x", 0, 5, 2),
param.suggest_loguniform("b/z", 1e-4, 1e-2)
]
sampler1 = RandomSampler(seed=1234)
searcher1 = OptunaSearch(space=converted_config,
sampler=sampler1,
metric="a",
mode="max")
sampler2 = RandomSampler(seed=1234)
searcher2 = OptunaSearch(space=optuna_config,
sampler=sampler2,
metric="a",
mode="max")
config1 = searcher1.suggest("0")
config2 = searcher2.suggest("0")
self.assertEqual(config1, config2)
self.assertIn(config1["a"], [2, 3, 4])
self.assertIn(config1["b"]["x"], list(range(5)))
self.assertLess(1e-4, config1["b"]["z"])
self.assertLess(config1["b"]["z"], 1e-2)
searcher = OptunaSearch(metric="a", mode="max")
analysis = tune.run(_mock_objective,
config=config,
search_alg=searcher,
num_samples=1)
trial = analysis.trials[0]
assert trial.config["a"] in [2, 3, 4]
mixed_config = {
"a": tune.uniform(5, 6),
"b": tune.uniform(8, 9) # Cannot mix List and Dict
}
searcher = OptunaSearch(space=mixed_config, metric="a", mode="max")
config = searcher.suggest("0")
self.assertTrue(5 <= config["a"] <= 6)
self.assertTrue(8 <= config["b"] <= 9)
parser.add_argument("--test", action="store_true")
args = parser.parse_args()
exp_name = args.exp_name
assert os.getenv("OMP_NUM_THREADS") == '1'
assert args.env_name in [
'Walker2DBulletEnv-v0', 'HalfCheetahBulletEnv-v0', 'AntBulletEnv-v0',
'HopperBulletEnv-v0', 'HumanoidBulletEnv-v0',
'HumanoidFlagrunBulletEnv-v0'
]
test = args.test
walker_config = {
DELAY_UPDATE: tune.grid_search([True]),
CONSTRAIN_NOVELTY: tune.grid_search(['soft']),
REPLAY_VALUES: tune.grid_search([False]),
USE_DIVERSITY_VALUE_NETWORK: tune.grid_search([True]),
"env": MultiAgentEnvWrapper,
"env_config": {
"env_name": args.env_name,
"num_agents": tune.grid_search([5])
},
# should be fixed
"kl_coeff": 1.0,
"gamma": 0.995,
"lambda": 0.95,
"clip_param": 0.2,
"num_sgd_iter": 20,
required=False,
help="The address of server to connect to if using "
"Ray Client.",
)
args, _ = parser.parse_known_args()
if args.server_address:
import ray
ray.init(f"ray://{args.server_address}")
config = {
"objective": "binary",
"metric": ["binary_error", "binary_logloss"],
"verbose": -1,
"boosting_type": tune.grid_search(["gbdt", "dart"]),
"num_leaves": tune.randint(10, 1000),
"learning_rate": tune.loguniform(1e-8, 1e-1),
}
analysis = tune.run(
train_breast_cancer,
metric="binary_error",
mode="min",
config=config,
num_samples=2,
scheduler=ASHAScheduler(),
)
print("Best hyperparameters found were: ", analysis.best_config)
from __future__ import print_function
import ray
from ray.tune import grid_search, register_env, run_experiments
from env import CarlaEnv, ENV_CONFIG
from models import register_carla_model
from scenarios import TOWN2_STRAIGHT
env_name = "carla_env"
env_config = ENV_CONFIG.copy()
env_config.update({
"verbose": False,
"x_res": 80,
"y_res": 80,
"squash_action_logits": grid_search([False, True]),
"use_depth_camera": False,
"discrete_actions": False,
"server_map": "/Game/Maps/Town02",
"reward_function": grid_search(["custom", "corl2017"]),
"scenarios": TOWN2_STRAIGHT,
})
register_env(env_name, lambda env_config: CarlaEnv(env_config))
register_carla_model()
redis_address = ray.services.get_node_ip_address() + ":6379"
run_experiments({
"carla-a3c": {
"run": "A3C",
"env": "carla_env",
},
"framework": args.framework,
# Use GPUs iff `RLLIB_NUM_GPUS` env var set to > 0.
"num_gpus": int(os.environ.get("RLLIB_NUM_GPUS", "0")),
}
group = False
elif args.run == "QMIX":
config = {
"rollout_fragment_length": 4,
"train_batch_size": 32,
"exploration_config": {
"epsilon_timesteps": 5000,
"final_epsilon": 0.05,
},
"num_workers": 0,
"mixer": grid_search([None, "qmix", "vdn"]),
"env_config": {
"separate_state_space": True,
"one_hot_state_encoding": True
},
# Use GPUs iff `RLLIB_NUM_GPUS` env var set to > 0.
"num_gpus": int(os.environ.get("RLLIB_NUM_GPUS", "0")),
}
group = True
else:
config = {
# Use GPUs iff `RLLIB_NUM_GPUS` env var set to > 0.
"num_gpus": int(os.environ.get("RLLIB_NUM_GPUS", "0")),
"framework": args.framework,
}
group = False
"""
def _build_layers_v2(self, input_dict, num_outputs, options):
print(input_dict)
self.obs_in = input_dict["obs"]
self.fcnet = FullyConnectedNetwork(
input_dict, self.obs_space, self.action_space, num_outputs, options
)
return self.fcnet.outputs, self.fcnet.last_layer
if __name__ == "__main__":
print("THIS EXPERIMENT HAS NOT BEEN FULLY TESTED")
kill_server()
ray.init()
ModelCatalog.register_custom_model("my_model", CustomModel)
tune.run(
"PPO",
stop={"timesteps_total": 1000000},
checkpoint_freq=1,
config={
"env": CarlaEnv, # CarlaEnv,SimpleCorridor, # or "corridor" if registered above
"model": {"custom_model": "my_model"},
"lr": grid_search([1e-2, 1e-4, 1e-6]), # try different lrs
"num_workers": 4, # parallelism
"num_gpus_per_worker": 0.2,
"env_config": env_config,
},
resume=False,
)
'--data_dir', type=str, default='/tmp/tensorflow/mnist/input_data',
help='Directory for storing input data')
FLAGS, unparsed = parser.parse_known_args()
tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)
# !!! Example of using the ray.tune Python API !!!
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument(
'--smoke-test', action='store_true', help='Finish quickly for testing')
args, _ = parser.parse_known_args()
register_trainable('train_mnist', train)
mnist_spec = {
'run': 'train_mnist',
'stop': {
'mean_accuracy': 0.99,
'time_total_s': 600,
},
'config': {
'activation': grid_search(['relu', 'elu', 'tanh']),
},
}
if args.smoke_test:
mnist_spec['stop']['training_iteration'] = 2
ray.init()
run_experiments({'tune_mnist_test': mnist_spec})
args, _ = parser.parse_known_args()
train_spec = {
"run": Cifar10Model,
"resources_per_trial": {
"cpu": 1,
"gpu": 1
},
"stop": {
"mean_accuracy": 0.80,
"training_iteration": 30,
},
"config": {
"epochs": 1,
"batch_size": 64,
"lr": grid_search([10**-4, 10**-5]),
"decay": sample_from(lambda spec: spec.config.lr / 100.0),
"dropout": grid_search([0.25, 0.5]),
},
"num_samples": 4,
}
if args.smoke_test:
train_spec["config"]["lr"] = 10**-4
train_spec["config"]["dropout"] = 0.5
ray.init()
pbt = PopulationBasedTraining(
time_attr="training_iteration",
reward_attr="mean_accuracy",
