def main(local_dir):
"""Main loop based on `rllib.examples.saving_experiences`."""
# pylint: disable=too-many-locals
batch_builder = SampleBatchBuilder()
writer = JsonWriter(local_dir)
env = _industrial_benchmark_maker({"max_episode_steps": 1000})
policy = IBBehaviorPolicy(env.observation_space, env.action_space, {})
for eps_id in trange(100):
obs = env.reset()
prev_action = np.zeros_like(env.action_space.sample())
prev_reward = 0
done = False
time = 0
while not done:
action, _, _ = policy.compute_single_action(obs, [])
new_obs, rew, done, info = env.step(action)
batch_builder.add_values(
t=time,
eps_id=eps_id,
agent_index=0,
obs=obs,
actions=action,
action_prob=1.0, # put the true action probability here
rewards=rew,
prev_actions=prev_action,
prev_rewards=prev_reward,
dones=done,
infos=info,
new_obs=new_obs,
)
obs = new_obs
prev_action = action
prev_reward = rew
time += 1
writer.write(batch_builder.build_and_reset())
self.observation[len(CONTEXT_ATTRIBUTES):] = action
return self.observation, reward, done, {}
def action_space_sample(self):
dist = np.random.randint(10)
change_topics = np.random.choice(N_TOPICS,dist,replace=False)
action = self.observation[len(CONTEXT_ATTRIBUTES):].copy()
for i in change_topics:
action[i] = 0 if action[i] else 1
return action
if __name__ == "__main__":
args = parser.parse_args()
batch_builder = SampleBatchBuilder()
output_path = os.path.join(ray.utils.get_user_temp_dir(), "demo-out")
writer = JsonWriter(output_path)
print("OUTPUT IN {}".format(output_path))
env = NewsWorld()
prep = get_preprocessor(env.observation_space)(env.observation_space)
print("The preprocessor is", prep)
for eps_id in range(args.stop):
obs = env.reset()
prev_action = np.zeros_like(env.action_space_sample())
prev_reward = 0
done = False
import pickle
import numpy as np
from ray.rllib.evaluation.sample_batch_builder import SampleBatchBuilder
from ray.rllib.offline.json_writer import JsonWriter
batch_builder = SampleBatchBuilder() # or MultiAgentSampleBatchBuilder
writer = JsonWriter("")
actions = [
'strength_1', 'strength_2', 'flexibility_1', 'flexibility_2', 'rest'
]
action_to_int_converter = {actions[i]: i for i in range(len(actions))}
#
with open('episode_actions.pkl', 'rb') as file:
episode_actions = pickle.load(file)
with open('episode_states.pkl', 'rb') as file:
episode_states = pickle.load(file)
n_episodes = len(episode_actions)
states0 = episode_states[0]
actions0 = episode_actions[0]
# import ipdb; ipdb.set_trace()
# save in batches
for episode_idx in range(n_episodes):
obs = np.array([0, 0, 0])
prev_action = None
prev_reward = None
done = False
n_actions = len(episode_actions[episode_idx])
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
"""Simple example of writing experiences to a file using JsonWriter."""
# __sphinx_doc_begin__
import gym
import numpy as np
from ray.rllib.evaluation.sample_batch_builder import SampleBatchBuilder
from ray.rllib.offline.json_writer import JsonWriter
if __name__ == "__main__":
batch_builder = SampleBatchBuilder() # or MultiAgentSampleBatchBuilder
writer = JsonWriter("/tmp/demo-out")
# You normally wouldn't want to manually create sample batches if a
# simulator is available, but let's do it anyways for example purposes:
env = gym.make("CartPole-v0")
for eps_id in range(100):
obs = env.reset()
prev_action = np.zeros_like(env.action_space.sample())
prev_reward = 0
done = False
t = 0
while not done:
action = env.action_space.sample()
new_obs, rew, done, info = env.step(action)
batch_builder.add_values(
t=t,
args = parser.parse_args()
ray.init()
cls = get_agent_class(args.run)
agent = cls(env=args.env)
agent.restore(args.checkpoint)
# TODO: Refactor this...
env = agent.workers.local_worker().env
if args.target_env:
env_creator = _global_registry.get(ENV_CREATOR, args.target_env)
target_env = env_creator(env.config)
else:
target_env = None
batch_builder = SampleBatchBuilder()
writer = JsonWriter(args.out)
for states_path in args.states:
with open(states_path, 'r') as states_file:
states = parse_states_file(states_file, env, args.skip_count)
trajectory = annotated_trajectory(states, agent, env, target_env)
prev_action = None
for (t, (obs, action, new_obs)) in enumerate(trajectory):
batch_builder.add_values(
t=t,
eps_id=0,
agent_index=0,
obs=obs,
actions=action,
action_prob=
1.0, # TODO: put the true action probability here
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
"""Simple example of writing experiences to a file using JsonWriter."""
# __sphinx_doc_begin__
import gym
import numpy as np
from ray.rllib.evaluation.sample_batch_builder import SampleBatchBuilder
from ray.rllib.offline.json_writer import JsonWriter
if __name__ == "__main__":
batch_builder = SampleBatchBuilder() # or MultiAgentSampleBatchBuilder
writer = JsonWriter("/tmp/demo-out")
# You normally wouldn't want to manually create sample batches if a
# simulator is available, but let's do it anyways for example purposes:
env = gym.make("CartPole-v0")
for eps_id in range(100):
obs = env.reset()
prev_action = np.zeros_like(env.action_space.sample())
prev_reward = 0
done = False
t = 0
while not done:
action = env.action_space.sample()
new_obs, rew, done, info = env.step(action)
batch_builder.add_values(
t=t,
def main():
args = parser.parse_args()
if args.save_path is None:
save_path = os.path.join(args.data_path, 'rllib')
else:
save_path = args.save_path
if args.env is None:
env_list = []
for env_spec in minerl.herobraine.envs.obfuscated_envs:
env_list.append(env_spec.name)
else:
env_list = [args.env]
register()
for env_name in env_list:
env = gym.make(env_name)
env = env.MineRLObservationWrapper(env.MineRLActionWrapper(env))
batch_builder = SampleBatchBuilder()
writer = JsonWriter(os.path.join(save_path, env_name))
prep = get_preprocessor(env.observation_space)(env.observation_space)
env.close()
data = minerl.data.make(env_name, data_dir=args.data_path)
for trajectory_name in data.get_trajectory_names():
t = 0
prev_action = None
prev_reward = 0
done = False
obs = None
info = None
for obs, action, reward, next_obs, done in data.load_data(
trajectory_name):
obs = (obs['pov'], obs['vector'])
next_obs = (next_obs['pov'], next_obs['vector'])
action = action['vector']
if prev_action is None:
prev_action = np.zeros_like(action)
batch_builder.add_values(
t=t,
eps_id=trajectory_name,
agent_index=0,
obs=prep.transform(obs),
actions=action,
action_prob=1.0, # put the true action probability here
rewards=reward,
prev_actions=prev_action,
prev_rewards=prev_reward,
dones=done,
infos=info,
new_obs=prep.transform(next_obs))
prev_action = action
prev_reward = reward
t += 1
writer.write(batch_builder.build_and_reset())
def write_jsons(
environment,
data_dir,
env_config,
save_path,
overwrite=False,
fail_safe=True,
**kwargs,
):
data_pipeline = minerl.data.make(environment, data_dir, **kwargs)
env = MinerRLDataEnv(data_pipeline)
env = wrap_env(env, env_config)
if not os.path.exists(save_path):
os.makedirs(save_path)
if len(os.listdir(save_path)) != 0:
abs_save_path = os.path.abspath(save_path)
if overwrite:
print(f"Overwriting! {abs_save_path}")
shutil.rmtree(abs_save_path)
else:
if fail_safe:
print(f"Json data already exists at {abs_save_path}")
return
else:
raise ValueError(
f"Directory {abs_save_path} not empty!"
f"Cannot overwrite existing data automatically, please delete old data if unused."
)
batch_builder = SampleBatchBuilder()
writer = JsonWriter(save_path)
prep = get_preprocessor(env.observation_space)(env.observation_space)
for eps_id, trajectory_name in enumerate(env.trajectory_names):
t = 0
prev_action = None
prev_reward = 0
done = False
try:
obs = env.reset()
except TypeError:
continue
while not done:
new_obs, reward, done, info = env.step(env.action_space.sample())
action = info["action"]
action = env.reverse_action(action)
if prev_action is None:
prev_action = np.zeros_like(action)
batch_builder.add_values(
t=t,
eps_id=eps_id,
agent_index=0,
obs=prep.transform(obs),
actions=action,
action_prob=1.0, # put the true action probability here
rewards=reward,
prev_actions=prev_action,
prev_rewards=prev_reward,
dones=done,
infos={"trajectory_name": trajectory_name},
new_obs=prep.transform(new_obs),
)
obs = new_obs
prev_action = action
prev_reward = reward
t += 1
writer.write(batch_builder.build_and_reset())
if window_still_open == False:
return False
if done:
break
if human_wants_restart:
break
while human_sets_pause:
env.wrapped.render()
time.sleep(0.1)
time.sleep(0.1)
print("timesteps %i reward %0.2f" % (total_timesteps, total_reward))
writer.write(batch_builder.build_and_reset())
if __name__ == "__main__":
batch_builder = SampleBatchBuilder() # or MultiAgentSampleBatchBuilder
writer = JsonWriter(DEMO_DATA_DIR)
env = MountainCar()
# RLlib uses preprocessors to implement transforms such as one-hot encoding
# and flattening of tuple and dict observations. For CartPole a no-op
# preprocessor is used, but this may be relevant for more complex envs.
prep = get_preprocessor(env.observation_space)(env.observation_space)
print("The preprocessor is", prep)
if not hasattr(env.action_space, "n"):
raise Exception("Keyboard agent only supports discrete action spaces")
ACTIONS = env.action_space.n
SKIP_CONTROL = 0 # Use previous control decision SKIP_CONTROL times, that's how you
# can test what skip is still usable.
def main(args):
try:
opts, args = getopt.getopt(args, "", ["sleep-for-animation=", ""])
except getopt.GetoptError as err:
print(str(err)) # will print something like "option -a not recognized"
sys.exit(2)
sleep_for_animation = True
for o, a in opts:
if o in ("--sleep-for-animation"):
sleep_for_animation = str2bool(a)
else:
assert False, "unhandled option"
batch_builder = SampleBatchBuilder() # or MultiAgentSampleBatchBuilder
writer = JsonWriter("./out/")
# Setting these 2 parameters to True can slow down training
visuals = False
sleep_for_animation = False
if visuals:
from flatland.utils.rendertools import RenderTool
max_depth = 30
tree_depth = 2
trial_start = 100
n_trials = 999
start = 0
columns = [
'Agents', 'X_DIM', 'Y_DIM', 'TRIAL_NO', 'REWARD', 'NORMALIZED_REWARD',
'DONE_RATIO', 'STEPS', 'ACTION_PROB'
]
df_all_results = pd.DataFrame(columns=columns)
for trials in range(trial_start, n_trials + 1):
env_file = f"envs-100-999/envs/Level_{trials}.pkl"
# env_file = f"../env_configs/round_1-small/Test_0/Level_{trials}.mpk"
# file = f"../env_configs/actions-small/Test_0/Level_{trials}.mpk"
file = f"envs-100-999/actions/envs/Level_{trials}.json"
if not os.path.isfile(env_file) or not os.path.isfile(file):
print("Missing file!", env_file, file)
continue
step = 0
obs_builder_object = TreeObsForRailEnv(
max_depth=tree_depth,
predictor=ShortestPathPredictorForRailEnv(max_depth))
env = RailEnv(
width=1,
height=1,
rail_generator=rail_from_file(env_file),
schedule_generator=schedule_from_file(env_file),
malfunction_generator_and_process_data=malfunction_from_file(
env_file),
obs_builder_object=obs_builder_object)
obs, info = env.reset(regenerate_rail=True,
regenerate_schedule=True,
activate_agents=False,
random_seed=1001)
with open(file, "r") as files:
expert_actions = json.load(files)
n_agents = env.get_num_agents()
x_dim, y_dim = env.width, env.height
agent_obs = [None] * n_agents
agent_obs_buffer = [None] * n_agents
done = dict()
done["__all__"] = False
if imitate:
agent_action_buffer = list(expert_actions[step].values())
else:
# , p=[0.2, 0, 0.5]) # [0] * n_agents
agent_action_buffer = np.random.choice(5, n_agents, replace=True)
update_values = [False] * n_agents
max_steps = int(4 * 2 * (20 + env.height + env.width))
action_size = 5 # 3
# And some variables to keep track of the progress
action_dict = dict()
scores_window = deque(maxlen=100)
reward_window = deque(maxlen=100)
done_window = deque(maxlen=100)
action_prob = [0] * action_size
# agent = Agent(state_size, action_size)
if visuals:
env_renderer = RenderTool(env, gl="PILSVG")
env_renderer.render_env(show=True,
frames=True,
show_observations=True)
for a in range(n_agents):
if obs[a]:
agent_obs[a] = normalize_observation(obs[a],
tree_depth,
observation_radius=10)
agent_obs_buffer[a] = agent_obs[a].copy()
# Reset score and done
score = 0
agent_action_buffer = np.zeros(n_agents)
# prev_action = np.zeros_like(envs.action_space.sample())
prev_reward = np.zeros(n_agents)
for step in range(max_steps):
for a in range(n_agents):
if info['action_required'][a]:
if imitate:
if step < len(expert_actions):
action = expert_actions[step][str(a)]
else:
action = 0
else:
action = 0
action_prob[action] += 1
update_values[a] = True
else:
update_values[a] = False
action = 0
action_dict.update({a: action})
next_obs, all_rewards, done, info = env.step(action_dict)
for a in range(n_agents):
if next_obs[a] is not None:
agent_obs[a] = normalize_observation(next_obs[a],
tree_depth,
observation_radius=10)
# Only update the values when we are done or when an action
# was taken and thus relevant information is present
if update_values[a] or done[a]:
start += 1
batch_builder.add_values(
t=step,
eps_id=trials,
agent_index=0,
obs=agent_obs_buffer[a],
actions=action_dict[a],
action_prob=1.0, # put the true action probability
rewards=all_rewards[a],
prev_actions=agent_action_buffer[a],
prev_rewards=prev_reward[a],
dones=done[a],
infos=info['action_required'][a],
new_obs=agent_obs[a])
agent_obs_buffer[a] = agent_obs[a].copy()
agent_action_buffer[a] = action_dict[a]
prev_reward[a] = all_rewards[a]
score += all_rewards[a] # / envs.get_num_agents()
if visuals:
env_renderer.render_env(show=True,
frames=True,
show_observations=True)
if sleep_for_animation:
time.sleep(0.5)
if done["__all__"] or step > max_steps:
writer.write(batch_builder.build_and_reset())
break
# Collection information about training
if step % 100 == 0:
tasks_finished = 0
for current_agent in env.agents:
if current_agent.status == RailAgentStatus.DONE_REMOVED:
tasks_finished += 1
print(
'\rTrial No {} Training {} Agents on ({},{}).\t Steps {}\t Reward: {:.3f}\t Normalized Reward: {:.3f}\tDones: {:.2f}%\t'
.format(
trials, env.get_num_agents(), x_dim, y_dim, step,
score, score / (max_steps + n_agents), 100 * np.mean(
tasks_finished / max(1, env.get_num_agents()))),
end=" ")
tasks_finished = 0
for current_agent in env.agents:
if current_agent.status == RailAgentStatus.DONE_REMOVED:
tasks_finished += 1
done_window.append(tasks_finished / max(1, env.get_num_agents()))
reward_window.append(score)
scores_window.append(score / (max_steps + n_agents))
data = [[
n_agents, x_dim, y_dim, trials,
np.mean(reward_window),
np.mean(scores_window), 100 * np.mean(done_window), step,
action_prob / np.sum(action_prob)
]]
df_cur = pd.DataFrame(data, columns=columns)
df_all_results = pd.concat([df_all_results, df_cur])
if imitate:
df_all_results.to_csv(
f'TreeImitationLearning_DQN_TrainingResults.csv', index=False)
print(
'\rTrial No {} Training {} Agents on ({},{}).\t Total Steps {}\t Reward: {:.3f}\t Normalized Reward: {:.3f}\tDones: {:.2f}%\t'
.format(trials, env.get_num_agents(), x_dim, y_dim, step,
np.mean(reward_window), np.mean(scores_window),
100 * np.mean(done_window)))
if visuals:
env_renderer.close_window()
gc.collect()