def launch_env(id=None):
env = None
if id is None:
from gym_duckietown.simulator import Simulator
env = Simulator(
seed=123, # random seed
map_name="loop_empty",
max_steps=500001, # we don't want the gym to reset itself
domain_rand=False,
camera_width=640,
camera_height=480,
accept_start_angle_deg=4, # start close to straight
full_transparency=True,
distortion=True,
)
else:
env = gym.make(id)
# Wrappers
env = ResizeWrapper(env)
env = NormalizeWrapper(env)
env = ImgWrapper(env) # to make the images from 160x120x3 into 3x160x120
env = SteeringToWheelVelWrapper(env)
env = ActionWrapper(env)
#env = DtRewardWrapper(env)
return env
def prepare_control_env(env, game, modify):
if 'n' in modify and type(env.observation_space) == gym.spaces.Box:
print('Normalizing input space')
env = NormalizeWrapper(env)
if 'r' in modify:
print('Reparametrizing the reward function')
env = ReparametrizeWrapper(env)
if 'p' in modify:
env = PILCOWrapper(env)
if 's' in modify:
print('Rescaled the reward function')
env = ScaleRewardWrapper(env)
if 'CartPole' in game:
env.observation_space = gym.spaces.Box(
np.array([-4.8, -10, -4.8, -10]), np.array([4.8, 10, 4.8, 10]))
return env
def solve(params, cis):
# python has dynamic typing, the line below can help IDEs with autocompletion
assert isinstance(cis, ChallengeInterfaceSolution)
# after this cis. will provide you with some autocompletion in some IDEs (e.g.: pycharm)
cis.info('Creating model.')
# you can have logging capabilties through the solution interface (cis).
# the info you log can be retrieved from your submission files.
# We get environment from the Evaluation Engine
cis.info('Making environment')
env = gym.make(params['env'])
# === BEGIN SUBMISSION ===
# If you created custom wrappers, you also need to copy them into this folder.
from wrappers import NormalizeWrapper, ImgWrapper, ActionWrapper, ResizeWrapper
env = ResizeWrapper(env)
env = NormalizeWrapper(env)
# to make the images pytorch-conv-compatible
env = ImgWrapper(env)
env = ActionWrapper(env)
# you ONLY need this wrapper if you trained your policy on [speed,steering angle]
# instead [left speed, right speed]
env = SteeringToWheelVelWrapper(env)
# you have to make sure that you're wrapping at least the actions
# and observations in the same as during training so that your model
# receives the same kind of input, because that's what it's trained for
# (for example if your model is trained on grayscale images and here
# you _don't_ make it grayscale too, then your model wont work)
# HERE YOU NEED TO CREATE THE POLICY NETWORK SAME AS YOU DID IN THE TRAINING CODE
# if you aren't using the DDPG baseline code, then make sure to copy your model
# into the model.py file and that it has a model.predict(state) method.
from model import DDPG
model = DDPG(state_dim=env.observation_space.shape,
action_dim=2,
max_action=1,
net_type="cnn")
try:
model.load("model", "models")
# === END SUBMISSION ===
# Then we make sure we have a connection with the environment and it is ready to go
cis.info('Reset environment')
observation = env.reset()
# While there are no signal of completion (simulation done)
# we run the predictions for a number of episodes, don't worry, we have the control on this part
while True:
# we passe the observation to our model, and we get an action in return
action = model.predict(observation)
# we tell the environment to perform this action and we get some info back in OpenAI Gym style
observation, reward, done, info = env.step(action)
# here you may want to compute some stats, like how much reward are you getting
# notice, this reward may no be associated with the challenge score.
# it is important to check for this flag, the Evalution Engine will let us know when should we finish
# if we are not careful with this the Evaluation Engine will kill our container and we will get no score
# from this submission
if 'simulation_done' in info:
cis.info('simulation_done received.')
break
if done:
cis.info('Episode done; calling reset()')
env.reset()
finally:
# release CPU/GPU resources, let's be friendly with other users that may need them
cis.info('Releasing resources')
try:
model.close()
except:
msg = 'Could not call model.close():\n%s' % traceback.format_exc()
cis.error(msg)
cis.info('Graceful exit of solve()')
:param rank: (int) index of the subprocess
"""
def _init():
return env
set_global_seeds(seed)
return _init
# Create the environment.
env_id = "Duckietown-loop_pedestrians-v0"
env = gym.make(env_id)
#env = WarpFrame(env)
env = ResizeWrapper(env) # Resize to 120, 160, 3
env = NormalizeWrapper(env)
#env = ImgWrapper(env) # to make the images from 160x120x3 into 3x160x120
env = ActionWrapper(env)
env = DtRewardWrapper(env)
env = DummyVecEnv([make_env(env_id, i) for i in range(1)])
env = VecFrameStack(env, n_stack=4)
# Model parameters.
total_timesteps = 1e6
model_type = "ddpg"
time_stamp = datetime.datetime.now().strftime("%Y%m%d-%H%M")
#parser.add_argument("--seed", default=0, type=int) # Sets Gym, PyTorch and Numpy seeds
#parser.add_argument("--start_timesteps", default=1e4, type=int) # How many time steps purely random policy is run for
#parser.add_argument("--eval_freq", default=5e3, type=float) # How often (time steps) we evaluate
#parser.add_argument("--expl_noise", default=0.1, type=float) # Std of Gaussian exploration noise
def _train(args):
if not os.path.exists("./results"):
os.makedirs("./results")
if not os.path.exists(args.model_dir):
os.makedirs(args.model_dir)
# Launch the env with our helper function
env = launch_env()
print("Initialized environment")
# Wrappers
env = ResizeWrapper(env)
env = NormalizeWrapper(env)
env = ImgWrapper(env) # to make the images from 160x120x3 into 3x160x120
env = ActionWrapper(env)
env = DtRewardWrapper(env)
print("Initialized Wrappers")
device = "cpu" # torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Set seeds
seed(args.seed)
state_dim = env.observation_space.shape
action_dim = env.action_space.shape[0]
max_action = float(env.action_space.high[0])
# Initialize policy
policy = DDPG(state_dim, action_dim, max_action, net_type="cnn")
replay_buffer = utils.ReplayBuffer(args.replay_buffer_max_size)
print("Initialized DDPG")
# Evaluate untrained policy
evaluations= [evaluate_policy(env, policy)]
total_timesteps = 0
timesteps_since_eval = 0
episode_num = 0
done = True
episode_reward = None
env_counter = 0
reward = 0
print("Starting training")
while total_timesteps < args.max_timesteps:
print("timestep: {} | reward: {}".format(total_timesteps, reward))
if done:
if total_timesteps != 0:
print(("Total T: %d Episode Num: %d Episode T: %d Reward: %f") % (
total_timesteps, episode_num, episode_timesteps, episode_reward))
policy.train(replay_buffer, episode_timesteps, args.batch_size, args.discount, args.tau)
# Evaluate episode
if timesteps_since_eval >= args.eval_freq:
timesteps_since_eval %= args.eval_freq
evaluations.append(evaluate_policy(env, policy))
print("rewards at time {}: {}".format(total_timesteps, evaluations[-1]))
if args.save_models:
policy.save(file_name, directory=args.model_dir)
np.savez("./results/{}.npz".format(file_name),evaluations)
# Reset environment
env_counter += 1
obs = env.reset()
done = False
episode_reward = 0
episode_timesteps = 0
episode_num += 1
# Select action randomly or according to policy
if total_timesteps < args.start_timesteps:
action = env.action_space.sample()
else:
action = policy.predict(np.array(obs))
if args.expl_noise != 0:
action = (action + np.random.normal(
0,
args.expl_noise,
size=env.action_space.shape[0])
).clip(env.action_space.low, env.action_space.high)
# Perform action
new_obs, reward, done, _ = env.step(action)
if episode_timesteps >= args.env_timesteps:
done = True
done_bool = 0 if episode_timesteps + 1 == args.env_timesteps else float(done)
episode_reward += reward
# Store data in replay buffer
replay_buffer.add(obs, new_obs, action, reward, done_bool)
obs = new_obs
episode_timesteps += 1
total_timesteps += 1
timesteps_since_eval += 1
print("Training done, about to save..")
policy.save(filename='ddpg', directory=args.model_dir)
print("Finished saving..should return now!")
def launch_local_experiment(environment):
# Use our launcher
env = launch_env(environment)
# === BEGIN SUBMISSION ===
# If you created custom wrappers, you also need to copy them into this folder.
from wrappers import NormalizeWrapper, ImgWrapper, ActionWrapper, ResizeWrapper
env = ResizeWrapper(env)
env = NormalizeWrapper(env)
# to make the images pytorch-conv-compatible
env = ImgWrapper(env)
env = ActionWrapper(env)
# you ONLY need this wrapper if you trained your policy on [speed,steering angle]
# instead [left speed, right speed]
# env = SteeringToWheelVelWrapper(env)
# you have to make sure that you're wrapping at least the actions
# and observations in the same as during training so that your model
# receives the same kind of input, because that's what it's trained for
# (for example if your model is trained on grayscale images and here
# you _don't_ make it grayscale too, then your model wont work)
# HERE YOU NEED TO CREATE THE POLICY NETWORK SAME AS YOU DID IN THE TRAINING CODE
# if you aren't using the DDPG baseline code, then make sure to copy your model
# into the model.py file and that it has a model.predict(state) method.
import model
my_model = model.model()
# === END SUBMISSION ===
# deactivate the automatic differentiation (i.e. the autograd engine, for calculating gradients)
observation = env.reset()
# While there are no signal of completion (simulation done)
# we run the predictions for a number of episodes, don't worry, we have the control on this part
trial = 0
reward =.0000001
rewards = defaultdict(list)
while trial < 1:
trial += 1
step = 0
while reward != -1000 and step != 300:
step += 1
# we passe the observation to our model, and we get an action in return
action = my_model.predict(observation, trial, step)
print("step: "+str(step))
print(action)
# we tell the environment to perform this action and we get some info back in OpenAI Gym style
observation, reward, done, info = env.step(action)
# here you may want to compute some stats, like how much reward are you getting
# notice, this reward may no be associated with the challenge score.
rewards[trial].append(reward)
# it is important to check for this flag, the Evalution Engine will let us know when should we finish
# if we are not careful with this the Evaluation Engine will kill our container and we will get no score
# from this submission
print("num_steps: " + str(float(len(rewards[trial]))))
print("mean reward: " + str(sum(rewards[trial]) / float(len(rewards[trial]))))
env.reset()
reward = 0
