def train_validate_env_generator(train_set, observation):
if train_set:
random_seed = np.random.randint(1000)
else:
random_seed = np.random.randint(1000, 2000)
test_env_no = np.random.randint(9)
level_no = np.random.randint(2)
random.seed(random_seed)
np.random.seed(random_seed)
test_envs_root = f"./test-envs/Test_{test_env_no}"
test_env_file_path = f"Level_{level_no}.pkl"
test_env_file_path = os.path.join(test_envs_root, test_env_file_path)
print(
f"Testing Environment: {test_env_file_path} with seed: {random_seed}")
env = RailEnv(width=1,
height=1,
rail_generator=rail_from_file(test_env_file_path),
schedule_generator=schedule_from_file(test_env_file_path),
malfunction_generator_and_process_data=malfunction_from_file(
test_env_file_path),
obs_builder_object=observation)
return env, random_seed
def env_create(self, obs_builder_object):
"""
Create a local env and remote env on which the
local agent can operate.
The observation builder is only used in the local env
and the remote env uses a DummyObservationBuilder
"""
time_start = time.time()
_request = {}
_request['type'] = messages.FLATLAND_RL.ENV_CREATE
_request['payload'] = {}
_response = self._remote_request(_request)
observation = _response['payload']['observation']
info = _response['payload']['info']
random_seed = _response['payload']['random_seed']
test_env_file_path = _response['payload']['env_file_path']
time_diff = time.time() - time_start
self.update_running_mean_stats("env_creation_wait_time", time_diff)
if not observation:
# If the observation is False,
# then the evaluations are complete
# hence return false
return observation, info
if self.verbose:
print("Received Env : ", test_env_file_path)
test_env_file_path = os.path.join(
self.test_envs_root,
test_env_file_path
)
if not os.path.exists(test_env_file_path):
raise Exception(
"\nWe cannot seem to find the env file paths at the required location.\n"
"Did you remember to set the AICROWD_TESTS_FOLDER environment variable "
"to point to the location of the Tests folder ? \n"
"We are currently looking at `{}` for the tests".format(self.test_envs_root)
)
if self.verbose:
print("Current env path : ", test_env_file_path)
self.current_env_path = test_env_file_path
self.env = RailEnv(width=1, height=1, rail_generator=rail_from_file(test_env_file_path),
schedule_generator=schedule_from_file(test_env_file_path),
malfunction_generator_and_process_data=malfunction_from_file(test_env_file_path),
obs_builder_object=obs_builder_object)
time_start = time.time()
local_observation, info = self.env.reset(
regenerate_rail=True,
regenerate_schedule=True,
activate_agents=False,
random_seed=random_seed
)
time_diff = time.time() - time_start
self.update_running_mean_stats("internal_env_reset_time", time_diff)
# Use the local observation
# as the remote server uses a dummy observation builder
return local_observation, info
def test_malfanction_to_and_from_file():
"""
Test loading malfunction from
Returns
-------
"""
stochastic_data = MalfunctionParameters(
malfunction_rate=1000, # Rate of malfunction occurence
min_duration=2, # Minimal duration of malfunction
max_duration=5 # Max duration of malfunction
)
rail, rail_map = make_simple_rail2()
env = RailEnv(
width=25,
height=30,
rail_generator=rail_from_grid_transition_map(rail),
schedule_generator=random_schedule_generator(),
number_of_agents=10,
malfunction_generator_and_process_data=malfunction_from_params(
stochastic_data))
env.reset()
#env.save("./malfunction_saving_loading_tests.pkl")
RailEnvPersister.save(env, "./malfunction_saving_loading_tests.pkl")
malfunction_generator, malfunction_process_data = malfunction_from_file(
"./malfunction_saving_loading_tests.pkl")
env2 = RailEnv(
width=25,
height=30,
rail_generator=rail_from_grid_transition_map(rail),
schedule_generator=random_schedule_generator(),
number_of_agents=10,
malfunction_generator_and_process_data=malfunction_from_params(
stochastic_data))
env2.reset()
assert env2.malfunction_process_data == env.malfunction_process_data
assert env2.malfunction_process_data.malfunction_rate == 1000
assert env2.malfunction_process_data.min_duration == 2
assert env2.malfunction_process_data.max_duration == 5
def handle_env_create(self, command):
"""
Handles a ENV_CREATE command from the client
TODO: Add a high level summary of everything thats happening here.
"""
self.simulation_count += 1
if self.simulation_count < len(self.env_file_paths):
"""
There are still test envs left that are yet to be evaluated
"""
test_env_file_path = self.env_file_paths[self.simulation_count]
print("Evaluating : {}".format(test_env_file_path))
test_env_file_path = os.path.join(self.test_env_folder,
test_env_file_path)
del self.env
self.env = RailEnv(
width=1,
height=1,
rail_generator=rail_from_file(test_env_file_path),
schedule_generator=schedule_from_file(test_env_file_path),
malfunction_generator_and_process_data=malfunction_from_file(
test_env_file_path),
obs_builder_object=DummyObservationBuilder())
if self.begin_simulation:
# If begin simulation has already been initialized
# atleast once
self.simulation_times.append(time.time() -
self.begin_simulation)
self.begin_simulation = time.time()
self.simulation_rewards.append(0)
self.simulation_rewards_normalized.append(0)
self.simulation_percentage_complete.append(0)
self.simulation_steps.append(0)
self.current_step = 0
_observation, _info = self.env.reset(regenerate_rail=True,
regenerate_schedule=True,
activate_agents=False,
random_seed=RANDOM_SEED)
if self.visualize:
if self.env_renderer:
del self.env_renderer
self.env_renderer = RenderTool(
self.env,
gl="PILSVG",
)
_command_response = {}
_command_response[
'type'] = messages.FLATLAND_RL.ENV_CREATE_RESPONSE
_command_response['payload'] = {}
_command_response['payload']['observation'] = _observation
_command_response['payload'][
'env_file_path'] = self.env_file_paths[self.simulation_count]
_command_response['payload']['info'] = _info
_command_response['payload']['random_seed'] = RANDOM_SEED
else:
"""
All test env evaluations are complete
"""
_command_response = {}
_command_response[
'type'] = messages.FLATLAND_RL.ENV_CREATE_RESPONSE
_command_response['payload'] = {}
_command_response['payload']['observation'] = False
_command_response['payload']['env_file_path'] = False
_command_response['payload']['info'] = False
_command_response['payload']['random_seed'] = False
self.send_response(_command_response, command)
#####################################################################
# Update evaluation state
#####################################################################
progress = np.clip(
self.simulation_count * 1.0 / len(self.env_file_paths), 0, 1)
mean_reward = round(np.mean(self.simulation_rewards), 2)
mean_normalized_reward = round(
np.mean(self.simulation_rewards_normalized), 2)
mean_percentage_complete = round(
np.mean(self.simulation_percentage_complete), 3)
self.evaluation_state["state"] = "IN_PROGRESS"
self.evaluation_state["progress"] = progress
self.evaluation_state["simulation_count"] = self.simulation_count
self.evaluation_state["score"]["score"] = mean_percentage_complete
self.evaluation_state["score"]["score_secondary"] = mean_reward
self.evaluation_state["meta"][
"normalized_reward"] = mean_normalized_reward
self.handle_aicrowd_info_event(self.evaluation_state)
skip = True
print("skip {} with {}".format(algo_name,
folder + "_" + file_name))
if skip:
continue
print("\n\n Instance " + folder + '/' + filename)
# Construct the enviornment from file
test = path + folder + '/' + filename
env = RailEnv(
width=1,
height=1,
rail_generator=rail_from_file(test),
schedule_generator=schedule_from_file(test),
malfunction_generator_and_process_data=malfunction_from_file(test),
)
env.reset()
CBS = PythonCBS(env, framework, "CBSH", timelimit, default_group_size,
debug, f_w, corridor_method, chasing,
accept_partial_solution, agent_priority_strategy)
success = CBS.search()
# Uncomment the following lines to write the paths to json files
# plan = CBS.getResult()
# with open(path + folder + "/" + file_name + ".json", 'w') as file:
# json.dump(plan, file)
# Uncomment the follwing lines to write detailed results to files for performance analysis
# dirName = "details"
from flatland.envs.agent_utils import RailAgentStatus
from flatland.utils.rendertools import RenderTool
import PIL
observation_tree_depth = 2
observation_radius = 10
observation_max_path_depth = 30
predictor = ShortestPathPredictorForRailEnv(observation_max_path_depth)
tree_observation = TreeObsForRailEnv(max_depth=observation_tree_depth,
predictor=predictor)
env_file = "D:/Sudhish/FYP/Final-Year-Project-main/Sudhish/envs-100-999/envs/Level_100.pkl"
env = RailEnv(
width=1,
height=1,
rail_generator=rail_from_file(env_file),
schedule_generator=schedule_from_file(env_file),
malfunction_generator_and_process_data=malfunction_from_file(env_file),
obs_builder_object=tree_observation)
obs, info = env.reset(True, True)
env_renderer = RenderTool(env, gl="PILSVG")
env_renderer.render_env()
image = env_renderer.get_image()
pil_image = PIL.Image.fromarray(image)
pil_image.show()
print("Env Loaded")
def main(args):
try:
opts, args = getopt.getopt(args, "", ["sleep-for-animation=", ""])
except getopt.GetoptError as err:
print(str(err)) # will print something like "option -a not recognized"
sys.exit(2)
sleep_for_animation = True
for o, a in opts:
if o in ("--sleep-for-animation"):
sleep_for_animation = str2bool(a)
else:
assert False, "unhandled option"
test_envs_root = f"./test-envs/Test_{test_env_no}"
test_env_file_path = f"Level_{level_no}.pkl"
test_env_file_path = os.path.join(
test_envs_root,
test_env_file_path
)
print(f"Testing Environment: {test_env_file_path} with seed: {random_seed}")
env = RailEnv(width=1, height=1, rail_generator=rail_from_file(test_env_file_path),
schedule_generator=schedule_from_file(test_env_file_path),
malfunction_generator_and_process_data=malfunction_from_file(test_env_file_path),
obs_builder_object=MultipleAgentNavigationObs(max_depth=2, predictor=ShortestPathPredictorForRailEnv(30)))
max_steps = int(4 * 2 * (20 + env.height + env.width))
obs, info = env.reset(regenerate_rail=True,
regenerate_schedule=True,
activate_agents=False,
random_seed=random_seed)
env_renderer = RenderTool(env, gl="PILSVG")
env_renderer.render_env(show=True, frames=True, show_observations=True)
n_agents = env.get_num_agents()
x_dim, y_dim = env.width,env.height
# Reset score and done
score = 0
env_done = 0
step = 0
for step in range(max_steps):
action_dict = {}
for i in range(n_agents):
if not obs:
action_dict.update({i: 2})
elif obs[i] is not None:
action = np.argmax(obs[i][1:4]) + 1
action_dict.update({i: action})
obs, all_rewards, done, _ = env.step(action_dict)
print("Rewards: ", all_rewards, " [done=", done, "]")
for a in range(env.get_num_agents()):
score += all_rewards[a] / env.get_num_agents()
env_renderer.render_env(show=True, frames=True, show_observations=True)
if sleep_for_animation:
time.sleep(0.5)
if done["__all__"]:
break
# Collection information about training
tasks_finished = 0
for current_agent in env.agents:
if current_agent.status == RailAgentStatus.DONE_REMOVED:
tasks_finished += 1
done_window = tasks_finished / max(1, env.get_num_agents())
scores_window = score / max_steps
print(
'\rTraining {} Agents on ({},{}).\t Steps {}\t Average Score: {:.3f}\tDones: {:.2f}%\t'.format(
n_agents, x_dim, y_dim,
step,
np.mean(scores_window),
100 * np.mean(done_window)), end=" ")
tasks_finished = 0
for current_agent in env.agents:
if current_agent.status == RailAgentStatus.DONE_REMOVED:
tasks_finished += 1
done_window = tasks_finished / max(1, env.get_num_agents())
scores_window = score / max_steps
print(
'\rTraining {} Agents on ({},{}).\t Total Steps {}\t Average Score: {:.3f}\tDones: {:.2f}%\t'.format(
n_agents, x_dim, y_dim,
step,
np.mean(scores_window),
100 * np.mean(done_window)), end=" ")
env_renderer.close_window()
def main(args):
try:
opts, args = getopt.getopt(args, "", ["sleep-for-animation=", ""])
except getopt.GetoptError as err:
print(str(err)) # will print something like "option -a not recognized"
sys.exit(2)
sleep_for_animation = True
for o, a in opts:
if o in ("--sleep-for-animation"):
sleep_for_animation = str2bool(a)
else:
assert False, "unhandled option"
batch_builder = SampleBatchBuilder() # or MultiAgentSampleBatchBuilder
writer = JsonWriter("./out/")
# Setting these 2 parameters to True can slow down training
visuals = False
sleep_for_animation = False
if visuals:
from flatland.utils.rendertools import RenderTool
max_depth = 30
tree_depth = 2
trial_start = 100
n_trials = 999
start = 0
columns = [
'Agents', 'X_DIM', 'Y_DIM', 'TRIAL_NO', 'REWARD', 'NORMALIZED_REWARD',
'DONE_RATIO', 'STEPS', 'ACTION_PROB'
]
df_all_results = pd.DataFrame(columns=columns)
for trials in range(trial_start, n_trials + 1):
env_file = f"envs-100-999/envs/Level_{trials}.pkl"
# env_file = f"../env_configs/round_1-small/Test_0/Level_{trials}.mpk"
# file = f"../env_configs/actions-small/Test_0/Level_{trials}.mpk"
file = f"envs-100-999/actions/envs/Level_{trials}.json"
if not os.path.isfile(env_file) or not os.path.isfile(file):
print("Missing file!", env_file, file)
continue
step = 0
obs_builder_object = TreeObsForRailEnv(
max_depth=tree_depth,
predictor=ShortestPathPredictorForRailEnv(max_depth))
env = RailEnv(
width=1,
height=1,
rail_generator=rail_from_file(env_file),
schedule_generator=schedule_from_file(env_file),
malfunction_generator_and_process_data=malfunction_from_file(
env_file),
obs_builder_object=obs_builder_object)
obs, info = env.reset(regenerate_rail=True,
regenerate_schedule=True,
activate_agents=False,
random_seed=1001)
with open(file, "r") as files:
expert_actions = json.load(files)
n_agents = env.get_num_agents()
x_dim, y_dim = env.width, env.height
agent_obs = [None] * n_agents
agent_obs_buffer = [None] * n_agents
done = dict()
done["__all__"] = False
if imitate:
agent_action_buffer = list(expert_actions[step].values())
else:
# , p=[0.2, 0, 0.5]) # [0] * n_agents
agent_action_buffer = np.random.choice(5, n_agents, replace=True)
update_values = [False] * n_agents
max_steps = int(4 * 2 * (20 + env.height + env.width))
action_size = 5 # 3
# And some variables to keep track of the progress
action_dict = dict()
scores_window = deque(maxlen=100)
reward_window = deque(maxlen=100)
done_window = deque(maxlen=100)
action_prob = [0] * action_size
# agent = Agent(state_size, action_size)
if visuals:
env_renderer = RenderTool(env, gl="PILSVG")
env_renderer.render_env(show=True,
frames=True,
show_observations=True)
for a in range(n_agents):
if obs[a]:
agent_obs[a] = normalize_observation(obs[a],
tree_depth,
observation_radius=10)
agent_obs_buffer[a] = agent_obs[a].copy()
# Reset score and done
score = 0
agent_action_buffer = np.zeros(n_agents)
# prev_action = np.zeros_like(envs.action_space.sample())
prev_reward = np.zeros(n_agents)
for step in range(max_steps):
for a in range(n_agents):
if info['action_required'][a]:
if imitate:
if step < len(expert_actions):
action = expert_actions[step][str(a)]
else:
action = 0
else:
action = 0
action_prob[action] += 1
update_values[a] = True
else:
update_values[a] = False
action = 0
action_dict.update({a: action})
next_obs, all_rewards, done, info = env.step(action_dict)
for a in range(n_agents):
if next_obs[a] is not None:
agent_obs[a] = normalize_observation(next_obs[a],
tree_depth,
observation_radius=10)
# Only update the values when we are done or when an action
# was taken and thus relevant information is present
if update_values[a] or done[a]:
start += 1
batch_builder.add_values(
t=step,
eps_id=trials,
agent_index=0,
obs=agent_obs_buffer[a],
actions=action_dict[a],
action_prob=1.0, # put the true action probability
rewards=all_rewards[a],
prev_actions=agent_action_buffer[a],
prev_rewards=prev_reward[a],
dones=done[a],
infos=info['action_required'][a],
new_obs=agent_obs[a])
agent_obs_buffer[a] = agent_obs[a].copy()
agent_action_buffer[a] = action_dict[a]
prev_reward[a] = all_rewards[a]
score += all_rewards[a] # / envs.get_num_agents()
if visuals:
env_renderer.render_env(show=True,
frames=True,
show_observations=True)
if sleep_for_animation:
time.sleep(0.5)
if done["__all__"] or step > max_steps:
writer.write(batch_builder.build_and_reset())
break
# Collection information about training
if step % 100 == 0:
tasks_finished = 0
for current_agent in env.agents:
if current_agent.status == RailAgentStatus.DONE_REMOVED:
tasks_finished += 1
print(
'\rTrial No {} Training {} Agents on ({},{}).\t Steps {}\t Reward: {:.3f}\t Normalized Reward: {:.3f}\tDones: {:.2f}%\t'
.format(
trials, env.get_num_agents(), x_dim, y_dim, step,
score, score / (max_steps + n_agents), 100 * np.mean(
tasks_finished / max(1, env.get_num_agents()))),
end=" ")
tasks_finished = 0
for current_agent in env.agents:
if current_agent.status == RailAgentStatus.DONE_REMOVED:
tasks_finished += 1
done_window.append(tasks_finished / max(1, env.get_num_agents()))
reward_window.append(score)
scores_window.append(score / (max_steps + n_agents))
data = [[
n_agents, x_dim, y_dim, trials,
np.mean(reward_window),
np.mean(scores_window), 100 * np.mean(done_window), step,
action_prob / np.sum(action_prob)
]]
df_cur = pd.DataFrame(data, columns=columns)
df_all_results = pd.concat([df_all_results, df_cur])
if imitate:
df_all_results.to_csv(
f'TreeImitationLearning_DQN_TrainingResults.csv', index=False)
print(
'\rTrial No {} Training {} Agents on ({},{}).\t Total Steps {}\t Reward: {:.3f}\t Normalized Reward: {:.3f}\tDones: {:.2f}%\t'
.format(trials, env.get_num_agents(), x_dim, y_dim, step,
np.mean(reward_window), np.mean(scores_window),
100 * np.mean(done_window)))
if visuals:
env_renderer.close_window()
gc.collect()
def handle_env_create(self, command):
"""
Handles a ENV_CREATE command from the client
TODO: Add a high level summary of everything thats happening here.
"""
if not self.simulation_done:
# trying to reset a simulation before finishing the previous one
_command_response = self._error_template(
"CAN'T CREATE NEW ENV BEFORE PREVIOUS IS DONE")
self.send_response(_command_response, command)
raise Exception(_command_response['payload'])
self.simulation_count += 1
self.simulation_done = False
if self.simulation_count < len(self.env_file_paths):
"""
There are still test envs left that are yet to be evaluated
"""
test_env_file_path = self.env_file_paths[self.simulation_count]
print("Evaluating : {}".format(test_env_file_path))
test_env_file_path = os.path.join(self.test_env_folder,
test_env_file_path)
del self.env
self.env = RailEnv(
width=1,
height=1,
rail_generator=rail_from_file(test_env_file_path),
schedule_generator=schedule_from_file(test_env_file_path),
malfunction_generator_and_process_data=malfunction_from_file(
test_env_file_path),
obs_builder_object=DummyObservationBuilder())
if self.begin_simulation:
# If begin simulation has already been initialized
# atleast once
# This adds the simulation time for the previous episode
self.simulation_times.append(time.time() -
self.begin_simulation)
self.begin_simulation = time.time()
# Update evaluation metadata for the previous episode
self.update_evaluation_metadata()
# Start adding placeholders for the new episode
self.simulation_env_file_paths.append(
os.path.relpath(test_env_file_path,
self.test_env_folder)) # relative path
self.simulation_rewards.append(0)
self.simulation_rewards_normalized.append(0)
self.simulation_percentage_complete.append(0)
self.simulation_steps.append(0)
self.current_step = 0
_observation, _info = self.env.reset(regenerate_rail=True,
regenerate_schedule=True,
activate_agents=False,
random_seed=RANDOM_SEED)
if self.visualize:
current_env_path = self.env_file_paths[self.simulation_count]
if current_env_path in self.video_generation_envs:
self.env_renderer = RenderTool(
self.env,
gl="PILSVG",
)
elif self.env_renderer:
self.env_renderer = False
_command_response = {}
_command_response[
'type'] = messages.FLATLAND_RL.ENV_CREATE_RESPONSE
_command_response['payload'] = {}
_command_response['payload']['observation'] = _observation
_command_response['payload'][
'env_file_path'] = self.env_file_paths[self.simulation_count]
_command_response['payload']['info'] = _info
_command_response['payload']['random_seed'] = RANDOM_SEED
else:
"""
All test env evaluations are complete
"""
_command_response = {}
_command_response[
'type'] = messages.FLATLAND_RL.ENV_CREATE_RESPONSE
_command_response['payload'] = {}
_command_response['payload']['observation'] = False
_command_response['payload']['env_file_path'] = False
_command_response['payload']['info'] = False
_command_response['payload']['random_seed'] = False
self.send_response(_command_response, command)
#####################################################################
# Update evaluation state
#####################################################################
progress = np.clip(
self.simulation_count * 1.0 / len(self.env_file_paths), 0, 1)
mean_reward, mean_normalized_reward, mean_percentage_complete = self.compute_mean_scores(
)
self.evaluation_state["state"] = "IN_PROGRESS"
self.evaluation_state["progress"] = progress
self.evaluation_state["simulation_count"] = self.simulation_count
self.evaluation_state["score"]["score"] = mean_percentage_complete
self.evaluation_state["score"]["score_secondary"] = mean_reward
self.evaluation_state["meta"][
"normalized_reward"] = mean_normalized_reward
self.handle_aicrowd_info_event(self.evaluation_state)
test_env_file_path = None
find_alternate_paths = True
observation_builder = GraphObsForRailEnv(
predictor=ShortestPathPredictorForRailEnv(
max_depth=max_prediction_depth),
bfs_depth=200)
if test_env_file_path:
env = RailEnv(
width=1,
height=1,
rail_generator=rail_from_file(test_env_file_path),
schedule_generator=schedule_from_file(test_env_file_path),
malfunction_generator_and_process_data=malfunction_from_file(
test_env_file_path),
obs_builder_object=observation_builder)
obs, _ = env.reset()
width = env.width
height = env.height
NUMBER_OF_AGENTS = env.number_of_agents
else:
rail_generator = sparse_rail_generator(max_num_cities=NUM_CITIES,
grid_mode=False,
max_rails_between_cities=3,
max_rails_in_city=4,
seed=1500)
env = RailEnv(width=width,
height=height,
