def launch_env(id=None):
env = None
if id is None:
from gym_duckietown.simulator import Simulator
env = Simulator(
seed=123, # random seed
map_name="loop_empty",
max_steps=500001, # we don't want the gym to reset itself
domain_rand=False,
camera_width=640,
camera_height=480,
accept_start_angle_deg=4, # start close to straight
full_transparency=True,
distortion=True,
)
else:
env = gym.make(id)
# Wrappers
env = ResizeWrapper(env)
env = NormalizeWrapper(env)
env = ImgWrapper(env) # to make the images from 160x120x3 into 3x160x120
env = SteeringToWheelVelWrapper(env)
env = ActionWrapper(env)
#env = DtRewardWrapper(env)
return env
def solve(params, cis):
# python has dynamic typing, the line below can help IDEs with autocompletion
assert isinstance(cis, ChallengeInterfaceSolution)
# after this cis. will provide you with some autocompletion in some IDEs (e.g.: pycharm)
cis.info('Creating model.')
# you can have logging capabilties through the solution interface (cis).
# the info you log can be retrieved from your submission files.
# We get environment from the Evaluation Engine
cis.info('Making environment')
env = gym.make(params['env'])
# === BEGIN SUBMISSION ===
# If you created custom wrappers, you also need to copy them into this folder.
from wrappers import NormalizeWrapper, ImgWrapper, ActionWrapper, ResizeWrapper
env = ResizeWrapper(env)
env = NormalizeWrapper(env)
# to make the images pytorch-conv-compatible
env = ImgWrapper(env)
env = ActionWrapper(env)
# you ONLY need this wrapper if you trained your policy on [speed,steering angle]
# instead [left speed, right speed]
env = SteeringToWheelVelWrapper(env)
# you have to make sure that you're wrapping at least the actions
# and observations in the same as during training so that your model
# receives the same kind of input, because that's what it's trained for
# (for example if your model is trained on grayscale images and here
# you _don't_ make it grayscale too, then your model wont work)
# HERE YOU NEED TO CREATE THE POLICY NETWORK SAME AS YOU DID IN THE TRAINING CODE
# if you aren't using the DDPG baseline code, then make sure to copy your model
# into the model.py file and that it has a model.predict(state) method.
from model import DDPG
model = DDPG(state_dim=env.observation_space.shape,
action_dim=2,
max_action=1,
net_type="cnn")
try:
model.load("model", "models")
# === END SUBMISSION ===
# Then we make sure we have a connection with the environment and it is ready to go
cis.info('Reset environment')
observation = env.reset()
# While there are no signal of completion (simulation done)
# we run the predictions for a number of episodes, don't worry, we have the control on this part
while True:
# we passe the observation to our model, and we get an action in return
action = model.predict(observation)
# we tell the environment to perform this action and we get some info back in OpenAI Gym style
observation, reward, done, info = env.step(action)
# here you may want to compute some stats, like how much reward are you getting
# notice, this reward may no be associated with the challenge score.
# it is important to check for this flag, the Evalution Engine will let us know when should we finish
# if we are not careful with this the Evaluation Engine will kill our container and we will get no score
# from this submission
if 'simulation_done' in info:
cis.info('simulation_done received.')
break
if done:
cis.info('Episode done; calling reset()')
env.reset()
finally:
# release CPU/GPU resources, let's be friendly with other users that may need them
cis.info('Releasing resources')
try:
model.close()
except:
msg = 'Could not call model.close():\n%s' % traceback.format_exc()
cis.error(msg)
cis.info('Graceful exit of solve()')
def _train(args):
if not os.path.exists("./results"):
os.makedirs("./results")
if not os.path.exists(args.model_dir):
os.makedirs(args.model_dir)
# Launch the env with our helper function
env = launch_env()
print("Initialized environment")
# Wrappers
env = ResizeWrapper(env)
env = NormalizeWrapper(env)
env = ImgWrapper(env) # to make the images from 160x120x3 into 3x160x120
env = ActionWrapper(env)
env = DtRewardWrapper(env)
print("Initialized Wrappers")
device = "cpu" # torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Set seeds
seed(args.seed)
state_dim = env.observation_space.shape
action_dim = env.action_space.shape[0]
max_action = float(env.action_space.high[0])
# Initialize policy
policy = DDPG(state_dim, action_dim, max_action, net_type="cnn")
replay_buffer = utils.ReplayBuffer(args.replay_buffer_max_size)
print("Initialized DDPG")
# Evaluate untrained policy
evaluations= [evaluate_policy(env, policy)]
total_timesteps = 0
timesteps_since_eval = 0
episode_num = 0
done = True
episode_reward = None
env_counter = 0
reward = 0
print("Starting training")
while total_timesteps < args.max_timesteps:
print("timestep: {} | reward: {}".format(total_timesteps, reward))
if done:
if total_timesteps != 0:
print(("Total T: %d Episode Num: %d Episode T: %d Reward: %f") % (
total_timesteps, episode_num, episode_timesteps, episode_reward))
policy.train(replay_buffer, episode_timesteps, args.batch_size, args.discount, args.tau)
# Evaluate episode
if timesteps_since_eval >= args.eval_freq:
timesteps_since_eval %= args.eval_freq
evaluations.append(evaluate_policy(env, policy))
print("rewards at time {}: {}".format(total_timesteps, evaluations[-1]))
if args.save_models:
policy.save(file_name, directory=args.model_dir)
np.savez("./results/{}.npz".format(file_name),evaluations)
# Reset environment
env_counter += 1
obs = env.reset()
done = False
episode_reward = 0
episode_timesteps = 0
episode_num += 1
# Select action randomly or according to policy
if total_timesteps < args.start_timesteps:
action = env.action_space.sample()
else:
action = policy.predict(np.array(obs))
if args.expl_noise != 0:
action = (action + np.random.normal(
0,
args.expl_noise,
size=env.action_space.shape[0])
).clip(env.action_space.low, env.action_space.high)
# Perform action
new_obs, reward, done, _ = env.step(action)
if episode_timesteps >= args.env_timesteps:
done = True
done_bool = 0 if episode_timesteps + 1 == args.env_timesteps else float(done)
episode_reward += reward
# Store data in replay buffer
replay_buffer.add(obs, new_obs, action, reward, done_bool)
obs = new_obs
episode_timesteps += 1
total_timesteps += 1
timesteps_since_eval += 1
print("Training done, about to save..")
policy.save(filename='ddpg', directory=args.model_dir)
print("Finished saving..should return now!")
file_name = "{}_{}_{}_{}".format(policy_name, experiment, args.exp_label,
args.seed)
# Launch the env with our helper function
env = launch_env(seed=111, map_name=args.map_name)
# Wrappers
env = wrappers.Monitor(env,
'./videos/test/' + file_name + '/',
force=True,
video_callable=lambda x: True)
env = CropWrapper(env)
env = ResizeWrapper(env)
#env = FrameStack(env, args.frame_stack_k)
env = NormalizeWrapper(env)
env = ImgWrapper(env) # to make the images from 160x120x3 into 3x160x120
env = SteeringToWheelVelWrapper(env)
#env = SoftActionWrapper(env)
#env = ActionWrapper(env)
#env = DtRewardWrapper(env) # not during testing
state_dim = env.observation_space.shape
action_dim = env.action_space.shape[0]
max_action = float(env.action_space.high[0])
# Initialize policy
policy = TD3(state_dim,
action_dim,
max_action,
net_type=args.net_type,
args=args)
def launch_local_experiment(environment):
# Use our launcher
env = launch_env(environment)
# === BEGIN SUBMISSION ===
# If you created custom wrappers, you also need to copy them into this folder.
from wrappers import NormalizeWrapper, ImgWrapper, ActionWrapper, ResizeWrapper
env = ResizeWrapper(env)
env = NormalizeWrapper(env)
# to make the images pytorch-conv-compatible
env = ImgWrapper(env)
env = ActionWrapper(env)
# you ONLY need this wrapper if you trained your policy on [speed,steering angle]
# instead [left speed, right speed]
# env = SteeringToWheelVelWrapper(env)
# you have to make sure that you're wrapping at least the actions
# and observations in the same as during training so that your model
# receives the same kind of input, because that's what it's trained for
# (for example if your model is trained on grayscale images and here
# you _don't_ make it grayscale too, then your model wont work)
# HERE YOU NEED TO CREATE THE POLICY NETWORK SAME AS YOU DID IN THE TRAINING CODE
# if you aren't using the DDPG baseline code, then make sure to copy your model
# into the model.py file and that it has a model.predict(state) method.
import model
my_model = model.model()
# === END SUBMISSION ===
# deactivate the automatic differentiation (i.e. the autograd engine, for calculating gradients)
observation = env.reset()
# While there are no signal of completion (simulation done)
# we run the predictions for a number of episodes, don't worry, we have the control on this part
trial = 0
reward =.0000001
rewards = defaultdict(list)
while trial < 1:
trial += 1
step = 0
while reward != -1000 and step != 300:
step += 1
# we passe the observation to our model, and we get an action in return
action = my_model.predict(observation, trial, step)
print("step: "+str(step))
print(action)
# we tell the environment to perform this action and we get some info back in OpenAI Gym style
observation, reward, done, info = env.step(action)
# here you may want to compute some stats, like how much reward are you getting
# notice, this reward may no be associated with the challenge score.
rewards[trial].append(reward)
# it is important to check for this flag, the Evalution Engine will let us know when should we finish
# if we are not careful with this the Evaluation Engine will kill our container and we will get no score
# from this submission
print("num_steps: " + str(float(len(rewards[trial]))))
print("mean reward: " + str(sum(rewards[trial]) / float(len(rewards[trial]))))
env.reset()
reward = 0