def run(args, parser):
if args.config_file:
with open(args.config_file) as f:
experiments = yaml.load(f)
else:
# Note: keep this in sync with tune/config_parser.py
experiments = {
args.experiment_name: { # i.e. log to ~/ray_results/default
"run": args.run,
"checkpoint_freq": args.checkpoint_freq,
"local_dir": args.local_dir,
"trial_resources": (
args.trial_resources and
resources_to_json(args.trial_resources)),
"stop": args.stop,
"config": dict(args.config, env=args.env),
"restore": args.restore,
"repeat": args.repeat,
"upload_dir": args.upload_dir,
}
}
for exp in experiments.values():
if not exp.get("run"):
parser.error("the following arguments are required: --run")
if not exp.get("env") and not exp.get("config", {}).get("env"):
parser.error("the following arguments are required: --env")
ray.init(redis_address=args.redis_address,
num_cpus=args.ray_num_cpus,
num_gpus=args.ray_num_gpus)
run_experiments(experiments,
scheduler=_make_scheduler(args),
queue_trials=args.queue_trials)
def run():
ModelCatalog.register_custom_model("my_vision_network", MyVisionNetwork)
config = {
"model": {
"custom_model": "my_vision_network",
"conv_filters": [[16, [5, 5], 4], [32, [3, 3], 1],
[256, [3, 3], 1]],
"custom_preprocessor": None
}
}
reb = RayExperimentBuilder(**config)
experiments, args, verbose = reb.get_experiment_definition()
ray.init(address=args.ray_address,
object_store_memory=args.ray_object_store_memory,
memory=args.ray_memory,
redis_max_memory=args.ray_redis_max_memory,
num_cpus=args.ray_num_cpus,
num_gpus=args.ray_num_gpus)
run_experiments(experiments,
scheduler=_make_scheduler(args),
queue_trials=args.queue_trials,
resume=args.resume,
verbose=verbose,
concurrent=True)
def run(args, parser):
with open(args.config_file) as f:
experiments = yaml.load(f)
if hasattr(args, 'restore'):
key = list(experiments.keys())[0]
experiments[key]['restore'] = args.restore
key = list(experiments.keys())[0]
experiments[key]["config"]["callbacks"] = {
"on_episode_end": tune.function(on_episode_end)
}
for exp in experiments.values():
if not exp.get("run"):
parser.error("the following arguments are required: --run")
if not exp.get("env") and not exp.get("config", {}).get("env"):
parser.error("the following arguments are required: --env")
ray.init(
redis_address=args.redis_address,
num_cpus=args.ray_num_cpus,
num_gpus=args.ray_num_gpus,
#object_store_memory=20000 * (2**20), # 100000 #int(25*10**9) #30gb
#redis_max_memory=5000 * (2**20)
)
run_experiments(experiments,
scheduler=_make_scheduler(args),
queue_trials=args.queue_trials)
def run(args, parser):
if args.config_file:
with open(args.config_file) as f:
experiments = yaml.safe_load(f)
else:
# Note: keep this in sync with tune/config_parser.py
experiments = {
args.experiment_name: { # i.e. log to ~/ray_results/default
"run": args.run,
"checkpoint_freq": args.checkpoint_freq,
"keep_checkpoints_num": args.keep_checkpoints_num,
"checkpoint_score_attr": args.checkpoint_score_attr,
"local_dir": args.local_dir,
"resources_per_trial": (
args.resources_per_trial and
resources_to_json(args.resources_per_trial)),
"stop": args.stop,
"config": dict(args.config, env=args.env),
"restore": args.restore,
"num_samples": args.num_samples,
"upload_dir": args.upload_dir,
}
}
for exp in experiments.values():
if not exp.get("run"):
parser.error("the following arguments are required: --run")
if not exp.get("env") and not exp.get("config", {}).get("env"):
parser.error("the following arguments are required: --env")
if args.eager:
exp["config"]["eager"] = True
if args.trace:
if not exp["config"].get("eager"):
raise ValueError("Must enable --eager to enable tracing.")
exp["config"]["eager_tracing"] = True
if args.ray_num_nodes:
cluster = Cluster()
for _ in range(args.ray_num_nodes):
cluster.add_node(num_cpus=args.ray_num_cpus or 1,
num_gpus=args.ray_num_gpus or 0,
object_store_memory=args.ray_object_store_memory,
memory=args.ray_memory,
redis_max_memory=args.ray_redis_max_memory)
ray.init(address=cluster.address)
else:
ray.init(address=args.ray_address,
object_store_memory=args.ray_object_store_memory,
memory=args.ray_memory,
redis_max_memory=args.ray_redis_max_memory,
num_cpus=args.ray_num_cpus,
num_gpus=args.ray_num_gpus)
run_experiments(experiments,
scheduler=_make_scheduler(args),
queue_trials=args.queue_trials,
resume=args.resume)
def launch(self):
"""Actual entry point into the class instance where everything happens."""
self.register_env_creator()
self.register_algorithms_and_preprocessors()
experiment_config, args, verbose = self.get_experiment_config()
# All worker nodes will block at this step during training
ray_cluster_config = self.ray_init_config()
if not self.is_master_node:
return
ray_custom_cluster_config = {
"object_store_memory": args.ray_object_store_memory,
"memory": args.ray_memory,
"redis_max_memory": args.ray_redis_max_memory,
"num_cpus": args.ray_num_cpus,
"num_gpus": args.ray_num_gpus,
}
all_workers_host_names = self.get_all_host_names()[1:]
# Overwrite redis address for single instance job
if len(all_workers_host_names) == 0:
ray_custom_cluster_config.update({"address": args.ray_address})
ray_cluster_config.update(ray_custom_cluster_config)
# Start the driver on master node
ray.init(**ray_cluster_config)
# Spot instance is back
if os.path.exists(CHECKPOINTS_DIR) and os.listdir(CHECKPOINTS_DIR):
print("Instance is back. Local checkpoint path detected.")
checkpoint_file = self.find_checkpoint_file_for_spot(
CHECKPOINTS_DIR)
print("Setting checkpoint path to {}".format(checkpoint_file))
if checkpoint_file:
experiment_config["training"][
"restore"] = checkpoint_file # Overwrite
experiment_config = self.customize_experiment_config(experiment_config)
experiment_config = self.set_up_checkpoint(experiment_config)
experiment_config["training"]["sync_to_driver"] = custom_sync_func
run_experiments(
experiment_config,
scheduler=_make_scheduler(args),
queue_trials=args.queue_trials,
resume=args.resume,
verbose=verbose,
concurrent=True,
)
# If distributed job, send TERMINATION_SIGNAL to all workers.
if len(all_workers_host_names) > 0:
self.sage_cluster_communicator.create_s3_signal(TERMINATION_SIGNAL)
def run(args, parser):
if args.config_file:
with open(args.config_file) as f:
experiments = yaml.load(f)
else:
# Note: keep this in sync with tune/config_parser.py
experiments = {
args.experiment_name: { # i.e. log to ~/ray_results/default
"run": args.run,
"checkpoint_freq": args.checkpoint_freq,
"local_dir": args.local_dir,
"resources_per_trial": (
args.resources_per_trial and
resources_to_json(args.resources_per_trial)),
"stop": args.stop,
"config": dict(args.config, env=args.env),
"restore": args.restore,
"num_samples": args.num_samples,
"upload_dir": args.upload_dir,
}
}
for exp in experiments.values():
if not exp.get("run"):
parser.error("the following arguments are required: --run")
if not exp.get("env") and not exp.get("config", {}).get("env"):
parser.error("the following arguments are required: --env")
if args.ray_num_nodes:
cluster = Cluster()
for _ in range(args.ray_num_nodes):
cluster.add_node(
num_cpus=args.ray_num_cpus or 1,
num_gpus=args.ray_num_gpus or 0,
object_store_memory=args.ray_object_store_memory,
redis_max_memory=args.ray_redis_max_memory)
ray.init(redis_address=cluster.redis_address)
else:
ray.init(
redis_address=args.redis_address,
object_store_memory=args.ray_object_store_memory,
redis_max_memory=args.ray_redis_max_memory,
num_cpus=args.ray_num_cpus,
num_gpus=args.ray_num_gpus)
run_experiments(
experiments,
scheduler=_make_scheduler(args),
queue_trials=args.queue_trials,
resume=args.resume)
def main(config_path: str) -> None:
ray.init()
env_type_map: Dict[str, Type] = {
"WarehouseSmall-v0": WarehouseSmallTrain,
"WarehouseMedium-v0": WarehouseMediumTrain,
"WarehouseLarge-v0": WarehouseLargeTrain,
}
for key, val in env_type_map.items():
register_env(key, lambda _: val())
with open(args.config_path) as config_file:
experiments = yaml.safe_load(config_file)
experiment_name = list(experiments.keys())[0]
experiments[experiment_name]["config"]["callbacks"] = {"on_episode_end": on_episode_end}
run_experiments(experiments)
if args.config_file:
with open(args.config_file) as f:
experiments = yaml.load(f)
else:
# Note: keep this in sync with tune/config_parser.py
experiments = {
args.experiment_name: { # i.e. log to /tmp/ray/default
"run": args.run,
"checkpoint_freq": args.checkpoint_freq,
"local_dir": args.local_dir,
"resources": resources_to_json(args.resources),
"stop": args.stop,
"config": dict(args.config, env=args.env),
"restore": args.restore,
"repeat": args.repeat,
"upload_dir": args.upload_dir,
}
}
for exp in experiments.values():
if not exp.get("run"):
parser.error("the following arguments are required: --run")
if not exp.get("env") and not exp.get("config", {}).get("env"):
parser.error("the following arguments are required: --env")
run_experiments(experiments,
scheduler=_make_scheduler(args),
redis_address=args.redis_address,
num_cpus=args.num_cpus,
num_gpus=args.num_gpus)
def run(args, parser):
if args.config_file:
with open(args.config_file) as f:
config_experiments = yaml.safe_load(f)
experiments = config_experiments
else:
if args.algo is not None:
args.experiment = args.algo
if args.experiment:
config_file = os.path.join('config', f'{args.experiment}.yaml')
with open(config_file) as f:
config_dict = yaml.safe_load(f)
else:
config_dict = {args.name: {}}
if args.debug:
args.env = 'MineRLRandomDebug-v0'
experiments = {}
for experiment_name, experiment_settings in config_dict.items():
config = dict(args.config, env=args.env)
# TODO: implement
if args.mode == 'offline':
config.update(
dict(
explore=False,
input=args.data_path,
input_evaluation=['simulation'],
))
elif args.mode == 'mixed':
config.update(
dict(
input={
args.data_path: args.mixing_ratio,
'sample': (1 - args.mixing_ratio)
},
input_evaluation=['simulation'],
))
if 'time_total_s' not in args.stop:
# The MineRL competition training time limit is 4 days. Subtract an hour for evaluation.
args.stop['time_total_s'] = int(2 * 24 * 60 * 60 -
3600) # limit two day training
if 'info/num_steps_sampled' not in args.stop:
# The MineRL competition environment sample limit is 8 million steps.
args.stop['info/num_steps_sampled'] = 8000000
if args.checkpoint_freq is None:
args.checkpoint_freq = 1000
if args.checkpoint_at_end is None:
args.checkpoint_at_end = True
if args.checkpoint_score_attr is None:
args.checkpoint_score_attr = 'episode_reward_mean'
# Note: keep this in sync with tune/config_parser.py
settings_from_args = { # i.e. log to ~/ray_results/default
"run": args.run,
"checkpoint_freq": args.checkpoint_freq,
"checkpoint_at_end": args.checkpoint_at_end,
"keep_checkpoints_num": args.keep_checkpoints_num,
"checkpoint_score_attr": args.checkpoint_score_attr,
"local_dir": args.local_dir,
"resources_per_trial": (
args.resources_per_trial and
resources_to_json(args.resources_per_trial)),
"stop": args.stop,
"config": config,
"restore": args.restore,
"num_samples": args.num_samples,
"upload_dir": args.upload_dir,
}
# overwrite the settings from arguments with those in the experiment config file
settings = merge_dicts(settings_from_args, experiment_settings)
experiments.update({experiment_name: settings})
if any('MineRL' in setting['config']['env']
for setting in experiments.values()):
import envs
envs.register(discrete=args.discrete,
num_actions=args.num_actions,
data_dir=args.data_dir)
print('\nArguments:')
pprint.pprint(args)
print('\nExperiment config:')
pprint.pprint(experiments)
print()
verbose = 1
for exp in experiments.values():
# Bazel makes it hard to find files specified in `args` (and `data`).
# Look for them here.
# NOTE: Some of our yaml files don't have a `config` section.
if exp.get("config", {}).get("input") and \
not os.path.exists(exp["config"]["input"]):
# This script runs in the ray/rllib dir.
rllib_dir = Path(__file__).parent
input_file = rllib_dir.absolute().joinpath(exp["config"]["input"])
exp["config"]["input"] = str(input_file)
if not exp.get("run"):
parser.error("the following arguments are required: --run")
if not exp.get("env") and not exp.get("config", {}).get("env"):
parser.error("the following arguments are required: --env")
if 'framework' not in exp['config']:
if args.eager:
exp["config"]["framework"] = "tfe"
elif args.torch:
exp["config"]["framework"] = "torch"
else:
exp["config"]["framework"] = "tf"
if args.v:
exp["config"]["log_level"] = "INFO"
verbose = 2
if args.vv:
exp["config"]["log_level"] = "DEBUG"
verbose = 3
if args.trace:
if exp["config"]["framework"] != "tfe":
raise ValueError("Must enable --eager to enable tracing.")
exp["config"]["eager_tracing"] = True
if args.ray_num_nodes:
cluster = Cluster()
for _ in range(args.ray_num_nodes):
cluster.add_node(num_cpus=args.ray_num_cpus or 1,
num_gpus=args.ray_num_gpus or 0,
object_store_memory=args.ray_object_store_memory,
memory=args.ray_memory,
redis_max_memory=args.ray_redis_max_memory)
ray.init(address=cluster.address)
else:
ray.init(include_dashboard=not args.no_ray_ui,
address=args.ray_address,
object_store_memory=args.ray_object_store_memory,
memory=args.ray_memory,
redis_max_memory=args.ray_redis_max_memory,
num_cpus=args.ray_num_cpus,
num_gpus=args.ray_num_gpus,
local_mode=args.local_mode)
run_experiments(experiments,
scheduler=_make_scheduler(args),
queue_trials=args.queue_trials,
resume=args.resume,
verbose=verbose,
concurrent=True)
ray.shutdown()
# "zero_mean": False,
# # Reduced channel depth and kernel size from default
# "conv_filters": [
# [32, [3, 3], 2],
# [32, [3, 3], 2],
# [32, [3, 3], 2],
# [32, [3, 3], 2],
# ]
# }
}
}
}
run_experiments(
experiments,
# raise_on_failed_trial=True,
# resume=True
)
# while True:
# result = agent.train()
# # print(result)
# print("training_iteration", result['training_iteration'])
# print("timesteps this iter", result['timesteps_this_iter'])
# print("timesteps_total", result['timesteps_total'])
# print("time_this_iter_s", result['time_this_iter_s'])
# print("time_total_s", result['time_total_s'])
# print("episode_reward_mean", result['episode_reward_mean'])
# print()
def run(args, parser):
if args.config_file:
with open(args.config_file) as f:
experiments = yaml.load(f)
else:
# Note: keep this in sync with tune/config_parser.py
experiments = {
args.experiment_name: { # i.e. log to ~/ray_results/default
"run": args.run,
"checkpoint_freq": args.checkpoint_freq,
"local_dir": args.local_dir,
"resources_per_trial": (
args.resources_per_trial and
resources_to_json(args.resources_per_trial)),
"stop": args.stop,
"config": dict(args.config, env=args.env),
"restore": args.restore,
"num_samples": args.num_samples,
"upload_dir": args.upload_dir,
}
}
# The default maximum number of bytes to allocate to the object store unless
# overridden by the user.
DEFAULT_OBJECT_STORE_MAX_MEMORY_BYTES = 20 * 10**9
# The smallest cap on the memory used by the object store that we allow.
OBJECT_STORE_MINIMUM_MEMORY_BYTES = 10**7
# The default maximum number of bytes that the non-primary Redis shards are
# allowed to use unless overridden by the user.
DEFAULT_REDIS_MAX_MEMORY_BYTES = 10**10
# The smallest cap on the memory used by Redis that we allow.
REDIS_MINIMUM_MEMORY_BYTES = 10**7
def on_episode_end(info):
episode = info["episode"]
env = info['env'].get_unwrapped()[0]
if hasattr(env, 'capital'):
capital_return = (env.capital -
env.initial_funds) / env.initial_funds
episode.custom_metrics['capital_return'] = capital_return
key = list(experiments.keys())[0]
experiments[key]["config"]["callbacks"] = {
"on_episode_end": tune.function(on_episode_end)
}
for exp in experiments.values():
if not exp.get("run"):
parser.error("the following arguments are required: --run")
if not exp.get("env") and not exp.get("config", {}).get("env"):
parser.error("the following arguments are required: --env")
if args.ray_num_nodes:
cluster = Cluster()
for _ in range(args.ray_num_nodes):
cluster.add_node(resources={
"num_cpus": args.ray_num_cpus or 1,
"num_gpus": args.ray_num_gpus or 0,
},
object_store_memory=args.ray_object_store_memory,
redis_max_memory=args.ray_redis_max_memory)
ray.init(redis_address=cluster.redis_address)
else:
print('init')
ray.init(redis_address=args.redis_address,
object_store_memory=int(0.5 * 10**9),
redis_max_memory=int(0.5 * 10**9),
num_cpus=args.ray_num_cpus,
num_gpus=args.ray_num_gpus)
run_experiments(experiments,
scheduler=_make_scheduler(args),
queue_trials=args.queue_trials,
resume=args.resume)
args = parser.parse_args(sys.argv[1:])
if args.config_file:
with open(args.config_file) as f:
experiments = yaml.load(f)
else:
# Note: keep this in sync with tune/config_parser.py
experiments = {
args.experiment_name: { # i.e. log to ~/ray_results/default
"run": args.run,
"checkpoint_freq": args.checkpoint_freq,
"local_dir": args.local_dir,
"resources": resources_to_json(args.resources),
"stop": args.stop,
"config": dict(args.config, env=args.env),
"restore": args.restore,
"repeat": args.repeat,
"upload_dir": args.upload_dir,
}
}
for exp in experiments.values():
if not exp.get("run"):
parser.error("the following arguments are required: --run")
if not exp.get("env") and not exp.get("config", {}).get("env"):
parser.error("the following arguments are required: --env")
ray.init(
redis_address=args.redis_address,
num_cpus=args.num_cpus, num_gpus=args.num_gpus)
run_experiments(experiments, scheduler=_make_scheduler(args))
else:
# Note: keep this in sync with tune/config_parser.py
experiments = {
args.experiment_name: { # i.e. log to ~/ray_results/default
"run": args.run,
"checkpoint_freq": args.checkpoint_freq,
"local_dir": args.local_dir,
"trial_resources": (
args.trial_resources and
resources_to_json(args.trial_resources)),
"stop": args.stop,
"config": dict(args.config, env=args.env),
"restore": args.restore,
"repeat": args.repeat,
"upload_dir": args.upload_dir,
}
}
for exp in experiments.values():
if not exp.get("run"):
parser.error("the following arguments are required: --run")
if not exp.get("env") and not exp.get("config", {}).get("env"):
parser.error("the following arguments are required: --env")
ray.init(redis_address=args.redis_address,
num_cpus=args.ray_num_cpus,
num_gpus=args.ray_num_gpus)
run_experiments(experiments,
scheduler=_make_scheduler(args),
queue_trials=args.queue_trials)
def run(args, parser):
if args.config_file:
with open(args.config_file) as f:
experiments = yaml.safe_load(f)
# add callbacks for self-defined metric
# and save successful transitions from RL agents
experiment_name = next(iter(experiments))
experiments[experiment_name]["config"]["optimizer"]["robot_demo_path"] = dir_path
experiments[experiment_name]["config"]["callbacks"] = {
"on_episode_start": on_episode_start,
"on_episode_step": on_episode_step,
"on_episode_end": on_episode_end,
"on_sample_end": on_sample_end,
"on_train_result": on_train_result,
"on_postprocess_traj": on_postprocess_traj
}
else:
# Note: keep this in sync with tune/config_parser.py
experiments = {
args.experiment_name: { # i.e. log to ~/ray_results/default
"run": args.run,
"checkpoint_freq": args.checkpoint_freq,
"keep_checkpoints_num": args.keep_checkpoints_num,
"checkpoint_score_attr": args.checkpoint_score_attr,
"local_dir": args.local_dir,
"resources_per_trial": (
args.resources_per_trial and
resources_to_json(args.resources_per_trial)),
"stop": args.stop,
"config": dict(args.config, env=args.env),
"restore": args.restore,
"num_samples": args.num_samples,
"upload_dir": args.upload_dir,
}
}
for exp in experiments.values():
if not exp.get("run"):
parser.error("the following arguments are required: --run")
if not exp.get("env") and not exp.get("config", {}).get("env"):
parser.error("the following arguments are required: --env")
if args.eager:
exp["config"]["eager"] = True
if args.trace:
if not exp["config"].get("eager"):
raise ValueError("Must enable --eager to enable tracing.")
exp["config"]["eager_tracing"] = True
if args.ray_num_nodes:
cluster = Cluster()
for _ in range(args.ray_num_nodes):
cluster.add_node(
num_cpus=args.ray_num_cpus or 1,
num_gpus=args.ray_num_gpus or 0,
object_store_memory=args.ray_object_store_memory,
memory=args.ray_memory,
redis_max_memory=args.ray_redis_max_memory)
ray.init(address=cluster.address) #, log_to_driver=False)
else:
ray.init(
address=args.ray_address,
object_store_memory=args.ray_object_store_memory,
memory=args.ray_memory,
redis_max_memory=args.ray_redis_max_memory,
num_cpus=args.ray_num_cpus,
num_gpus=args.ray_num_gpus)
# log_to_driver=False) # disable the loggings
# https://github.com/ray-project/ray/issues/5048
run_experiments(
experiments,
scheduler=_make_scheduler(args),
queue_trials=args.queue_trials,
resume=args.resume)
def run(args, parser):
# create exps from configs
if args.config_file:
# load configs from yaml
with open(args.config_file) as f:
exps = yaml.safe_load(f)
else:
exps = create_exps(args=args, )
arena_exps = create_arena_exps(
exps=exps,
args=args,
parser=parser,
)
# config ray cluster
if args.ray_num_nodes:
cluster = Cluster()
for ray_node in range(args.ray_num_nodes):
cluster.add_node(
num_cpus=args.ray_num_cpus or 1,
num_gpus=args.ray_num_gpus or 0,
object_store_memory=args.ray_object_store_memory,
memory=args.ray_memory,
redis_max_memory=args.ray_redis_max_memory,
)
ray.init(address=cluster.redis_address, )
else:
ray.init(
address=args.ray_address,
object_store_memory=args.ray_object_store_memory,
memory=args.ray_memory,
redis_max_memory=args.ray_redis_max_memory,
num_cpus=args.ray_num_cpus,
num_gpus=args.ray_num_gpus,
)
if len(arena_exps.keys()) > 1:
logger.warning(
"There are multiple experiments scheduled, ray==0.7.4 will run them one by one, instead of cocurrently. "
"However, recent ray can run them cocurrently. But the recent ray has failed our test (the rllib is broken)"
"This is mainly due to there are grid search used in configs that is not supported by original rllib. "
)
if args.eval:
# evaluate policies
if len(arena_exps.keys()) < 1:
raise ValueError
elif len(arena_exps.keys()) >= 1:
if len(arena_exps.keys()) > 1:
arena_exp_key = inquire_select(
choices=list(arena_exps.keys()),
key="arena_exp_key",
)
else:
# if there is just one arena_exps
arena_exp_key = list(arena_exps.keys())[0]
logger.info("Evaluating arena_exp_key: {}".format(arena_exp_key, ))
arena_exp = arena_exps[arena_exp_key]
answers = prompt(
[{
'type': 'input',
'name': 'eval_log_path',
'message':
'Where do you want to log the results of this evaluation?',
'default': '../eval_log_path/'
}],
style=custom_style_2,
)
prepare_path(answers['eval_log_path'])
from ray.rllib.evaluation.rollout_worker import RolloutWorker
# worker = ArenaRolloutWorker(
# TODO: RolloutWorker does not support monitor for multi-agent envs
worker = RolloutWorker(
env_creator=lambda _: ArenaRllibEnv(
env=arena_exp["env"],
env_config=arena_exp["config"]["env_config"],
),
policy=arena_exp["config"]["multiagent"]["policies"],
policy_mapping_fn=arena_exp["config"]["multiagent"]
["policy_mapping_fn"],
batch_mode="complete_episodes",
batch_steps=500,
num_envs=1,
monitor_path=answers['eval_log_path'],
)
logger.info("Testing worker...")
sample_start = time.time()
worker.sample()
sample_time = time.time() - sample_start
logger.info("Finish testing worker.")
policy_ids = list(worker.policy_map.keys())
checkpoints = inquire_checkpoints(
local_dir=arena_exp["local_dir"],
policy_ids=policy_ids,
)
checkpoint_paths = checkpoints_2_checkpoint_paths(checkpoints)
num_checkpoint_paths = {}
for policy_id, checkpoint_paths_per_policy_id in checkpoint_paths.items(
):
num_checkpoint_paths[policy_id] = len(
checkpoint_paths_per_policy_id)
num_sampling = np.prod(list(num_checkpoint_paths.values()))
confirm = inquire_confirm(
"You have scheduled {} sampling, each sampling will take {} minutes, which means {} hours in total."
.format(
num_sampling,
sample_time / 60.0,
num_sampling * sample_time / 60.0 / 60.0,
))
if not confirm:
os.exit()
result_matrix = run_result_matrix(
checkpoint_paths=checkpoint_paths,
worker=worker,
)
result_matrix = np.asarray(result_matrix)
vis_result_matrix(
result_matrix=result_matrix,
log_path=answers['eval_log_path'],
)
else:
run_experiments(
arena_exps,
scheduler=_make_scheduler(args),
queue_trials=args.queue_trials,
resume=args.resume,
)
def run(args, parser):
if args.config_file:
with open(args.config_file) as f:
experiments = yaml.safe_load(f)
else:
# Note: keep this in sync with tune/config_parser.py
experiments = {
args.experiment_name: { # i.e. log to ~/ray_results/default
"run": args.run,
"checkpoint_freq": args.checkpoint_freq,
"checkpoint_at_end": args.checkpoint_at_end,
"keep_checkpoints_num": args.keep_checkpoints_num,
"checkpoint_score_attr": args.checkpoint_score_attr,
"local_dir": args.local_dir,
"resources_per_trial": (
args.resources_per_trial and
resources_to_json(args.resources_per_trial)),
"stop": args.stop,
"config": dict(args.config, env=args.env),
"restore": args.restore,
"num_samples": args.num_samples,
"upload_dir": args.upload_dir,
}
}
verbose = 1
for exp in experiments.values():
# Bazel makes it hard to find files specified in `args` (and `data`).
# Look for them here.
# NOTE: Some of our yaml files don't have a `config` section.
if exp.get("config", {}).get("input") and \
not os.path.exists(exp["config"]["input"]):
# This script runs in the ray/rllib dir.
rllib_dir = Path(__file__).parent
input_file = rllib_dir.absolute().joinpath(exp["config"]["input"])
exp["config"]["input"] = str(input_file)
if not exp.get("run"):
parser.error("the following arguments are required: --run")
if not exp.get("env") and not exp.get("config", {}).get("env"):
parser.error("the following arguments are required: --env")
if args.torch:
exp["config"]["framework"] = "torch"
elif args.eager:
exp["config"]["framework"] = "tfe"
if args.trace:
if exp["config"]["framework"] not in ["tf2", "tfe"]:
raise ValueError("Must enable --eager to enable tracing.")
exp["config"]["eager_tracing"] = True
if args.v:
exp["config"]["log_level"] = "INFO"
verbose = 3 # Print details on trial result
if args.vv:
exp["config"]["log_level"] = "DEBUG"
verbose = 3 # Print details on trial result
if args.ray_num_nodes:
cluster = Cluster()
for _ in range(args.ray_num_nodes):
cluster.add_node(num_cpus=args.ray_num_cpus or 1,
num_gpus=args.ray_num_gpus or 0,
object_store_memory=args.ray_object_store_memory)
ray.init(address=cluster.address)
else:
ray.init(include_dashboard=not args.no_ray_ui,
address=args.ray_address,
object_store_memory=args.ray_object_store_memory,
num_cpus=args.ray_num_cpus,
num_gpus=args.ray_num_gpus,
local_mode=args.local_mode)
if IS_NOTEBOOK:
progress_reporter = JupyterNotebookReporter(
overwrite=verbose >= 3, print_intermediate_tables=verbose >= 1)
else:
progress_reporter = CLIReporter(print_intermediate_tables=verbose >= 1)
run_experiments(experiments,
scheduler=create_scheduler(args.scheduler,
**args.scheduler_config),
resume=args.resume,
queue_trials=args.queue_trials,
verbose=verbose,
progress_reporter=progress_reporter,
concurrent=True)
ray.shutdown()
def run(args, parser):
if args.config_file:
with open(args.config_file) as f:
experiments = yaml.safe_load(f)
else:
# Note: keep this in sync with tune/config_parser.py
experiments = {
args.experiment_name: { # i.e. log to ~/ray_results/default
"run": args.run,
"checkpoint_freq": args.checkpoint_freq,
"checkpoint_at_end": args.checkpoint_at_end,
"keep_checkpoints_num": args.keep_checkpoints_num,
"checkpoint_score_attr": args.checkpoint_score_attr,
"local_dir": args.local_dir,
"resources_per_trial": (
args.resources_per_trial
and resources_to_json(args.resources_per_trial)
),
"stop": args.stop,
"config": dict(args.config, env=args.env),
"restore": args.restore,
"num_samples": args.num_samples,
"sync_config": {
"upload_dir": args.upload_dir,
},
}
}
# Ray UI.
if args.no_ray_ui:
deprecation_warning(old="--no-ray-ui", new="--ray-ui", error=False)
args.ray_ui = False
verbose = 1
for exp in experiments.values():
# Bazel makes it hard to find files specified in `args` (and `data`).
# Look for them here.
# NOTE: Some of our yaml files don't have a `config` section.
input_ = exp.get("config", {}).get("input")
if input_ and input_ != "sampler":
# This script runs in the ray/rllib dir.
rllib_dir = Path(__file__).parent
def patch_path(path):
if isinstance(path, list):
return [patch_path(i) for i in path]
elif isinstance(path, dict):
return {
patch_path(k): patch_path(v)
for k, v in path.items()
}
elif isinstance(path, str):
if os.path.exists(path):
return path
else:
abs_path = str(rllib_dir.absolute().joinpath(path))
return abs_path if os.path.exists(abs_path) else path
else:
return path
exp["config"]["input"] = patch_path(input_)
if not exp.get("run"):
parser.error("the following arguments are required: --run")
if not exp.get("env") and not exp.get("config", {}).get("env"):
parser.error("the following arguments are required: --env")
if args.torch:
deprecation_warning("--torch", "--framework=torch")
exp["config"]["framework"] = "torch"
elif args.eager:
deprecation_warning("--eager", "--framework=[tf2|tfe]")
exp["config"]["framework"] = "tfe"
elif args.framework is not None:
exp["config"]["framework"] = args.framework
if args.trace:
if exp["config"]["framework"] not in ["tf2", "tfe"]:
raise ValueError("Must enable --eager to enable tracing.")
exp["config"]["eager_tracing"] = True
if args.v:
exp["config"]["log_level"] = "INFO"
verbose = 3 # Print details on trial result
if args.vv:
exp["config"]["log_level"] = "DEBUG"
verbose = 3 # Print details on trial result
if args.ray_num_nodes:
# Import this only here so that train.py also works with
# older versions (and user doesn't use `--ray-num-nodes`).
from ray.cluster_utils import Cluster
cluster = Cluster()
for _ in range(args.ray_num_nodes):
cluster.add_node(
num_cpus=args.ray_num_cpus or 1,
num_gpus=args.ray_num_gpus or 0,
object_store_memory=args.ray_object_store_memory,
)
ray.init(address=cluster.address)
else:
ray.init(
include_dashboard=args.ray_ui,
address=args.ray_address,
object_store_memory=args.ray_object_store_memory,
num_cpus=args.ray_num_cpus,
num_gpus=args.ray_num_gpus,
local_mode=args.local_mode,
)
if IS_NOTEBOOK:
progress_reporter = JupyterNotebookReporter(
overwrite=verbose >= 3, print_intermediate_tables=verbose >= 1)
else:
progress_reporter = CLIReporter(print_intermediate_tables=verbose >= 1)
run_experiments(
experiments,
scheduler=create_scheduler(args.scheduler, **args.scheduler_config),
resume=args.resume,
verbose=verbose,
progress_reporter=progress_reporter,
concurrent=True,
)
ray.shutdown()
def run(args, parser):
if args.config_file:
with open(args.config_file) as f:
experiments = yaml.safe_load(f)
else:
# Note: keep this in sync with tune/config_parser.py
experiments = {
args.experiment_name: { # i.e. log to ~/ray_results/default
"run": args.run,
"env": "HuskyPickAndPlace-v1",
"checkpoint_freq": 100, # args.checkpoint_freq,
"keep_checkpoints_num": args.keep_checkpoints_num,
"checkpoint_score_attr": args.checkpoint_score_attr,
"local_dir": args.local_dir,
"resources_per_trial": (
args.resources_per_trial and
resources_to_json(args.resources_per_trial)),
# "stop": args.stop,
"stop": {"timesteps_total": 4000000}, # 10M "episode_reward_mean": 18.0
# "config": {dict(args.config, env=args.env)},
"config": {
"num_workers": 10,
"ignore_worker_failures": True,
# "seed": 789,
"callbacks": {
"on_episode_start": on_episode_start,
"on_episode_step": on_episode_step,
"on_episode_end": on_episode_end,
},
},
"restore": args.restore,
"num_samples": args.num_samples,
"upload_dir": args.upload_dir,
}
}
# verbose = 1
# for exp in experiments.values():
# if not exp.get("run"):
# parser.error("the following arguments are required: --run")
# if not exp.get("env") and not exp.get("config", {}).get("env"):
# parser.error("the following arguments are required: --env")
# if args.eager:
# exp["config"]["eager"] = True
# if args.v:
# exp["config"]["log_level"] = "INFO"
# verbose = 2
# if args.vv:
# exp["config"]["log_level"] = "DEBUG"
# verbose = 3
# if args.trace:
# if not exp["config"].get("eager"):
# raise ValueError("Must enable --eager to enable tracing.")
# exp["config"]["eager_tracing"] = True
# if args.ray_num_nodes:
# cluster = Cluster()
# for _ in range(args.ray_num_nodes):
# cluster.add_node(
# num_cpus=args.ray_num_cpus or 1,
# num_gpus=args.ray_num_gpus or 0,
# object_store_memory=args.ray_object_store_memory,
# memory=args.ray_memory,
# redis_max_memory=args.ray_redis_max_memory)
# ray.init(address=cluster.address)
# else:
ray.init(address=args.ray_address,
object_store_memory=args.ray_object_store_memory,
memory=args.ray_memory,
redis_max_memory=args.ray_redis_max_memory,
num_cpus=args.ray_num_cpus,
num_gpus=args.ray_num_gpus)
run_experiments(
experiments,
scheduler=_make_scheduler(args),
queue_trials=args.queue_trials,
resume=args.resume,
verbose=2,
)
def run(args, parser):
if args.config_file:
with open(args.config_file) as f:
experiments = yaml.safe_load(f)
else:
# Note: keep this in sync with tune/config_parser.py
experiments = {
args.experiment_name: { # i.e. log to ~/ray_results/default
"run": args.run,
"checkpoint_freq": args.checkpoint_freq,
"keep_checkpoints_num": args.keep_checkpoints_num,
"checkpoint_score_attr": args.checkpoint_score_attr,
"local_dir": args.local_dir,
"resources_per_trial": (
args.resources_per_trial and
resources_to_json(args.resources_per_trial)),
"stop": args.stop,
"config": dict(args.config, env=args.env),
"restore": args.restore,
"num_samples": args.num_samples,
"upload_dir": args.upload_dir,
}
}
verbose = 1
for exp in experiments.values():
# Bazel makes it hard to find files specified in `args` (and `data`).
# Look for them here.
# NOTE: Some of our yaml files don't have a `config` section.
if exp.get("config", {}).get("input") and \
not os.path.exists(exp["config"]["input"]):
# This script runs in the ray/rllib dir.
rllib_dir = Path(__file__).parent
input_file = rllib_dir.absolute().joinpath(exp["config"]["input"])
exp["config"]["input"] = str(input_file)
if not exp.get("run"):
parser.error("the following arguments are required: --run")
if not exp.get("env") and not exp.get("config", {}).get("env"):
parser.error("the following arguments are required: --env")
if args.eager:
exp["config"]["eager"] = True
if args.torch:
exp["config"]["use_pytorch"] = True
if args.v:
exp["config"]["log_level"] = "INFO"
verbose = 2
if args.vv:
exp["config"]["log_level"] = "DEBUG"
verbose = 3
if args.trace:
if not exp["config"].get("eager"):
raise ValueError("Must enable --eager to enable tracing.")
exp["config"]["eager_tracing"] = True
### Add Custom Callbacks
exp["config"]["callbacks"] = CustomCallbacks
if args.ray_num_nodes:
cluster = Cluster()
for _ in range(args.ray_num_nodes):
cluster.add_node(num_cpus=args.ray_num_cpus or 1,
num_gpus=args.ray_num_gpus or 0,
object_store_memory=args.ray_object_store_memory,
memory=args.ray_memory,
redis_max_memory=args.ray_redis_max_memory)
ray.init(address=cluster.address)
else:
ray.init(address=args.ray_address,
object_store_memory=args.ray_object_store_memory,
memory=args.ray_memory,
redis_max_memory=args.ray_redis_max_memory,
num_cpus=args.ray_num_cpus,
num_gpus=args.ray_num_gpus)
# NOTE: customs
for exp in experiments.values():
exp["loggers"] = make_loggers(args)
# launch training
run_experiments(experiments,
scheduler=_make_scheduler(args),
queue_trials=args.queue_trials,
resume=args.resume,
verbose=verbose,
concurrent=True)
from ray.tune.tune import run_experiments
from ray.tune.suggest.variant_generator import grid_search
import random
experiment = {
'cartpole-ppo': {
'run': 'DQN',
'env': 'CartPole-v0',
# 'resources_per_trial': {
# 'cpu': 1,
# 'gpu': 0},
'stop': {
'episode_reward_mean': 200,
'time_total_s': 180
},
'config': {
'num_sgd_iter': grid_search([1, 4]),
'num_workers': 2,
# 'use_critic' : random.choice([True, False]),
# "train_batch_size": random.choice([1024, 2048, 4096]),
"lr": [1e-3, 5e-4, 1e-4, 5e-5, 1e-5],
# 'sgd_minibatch_size': grid_search([128, 256, 512])
}
},
# put additional experiments to run concurrently here
}
ray.init()
run_experiments(experiment)
