def setup(difficulty_level='default', env_name = "AirSimEnv-v42"):
#parser = argparse.ArgumentParser()
#parser.add_argument('--mode', choices=['train', 'test'], default='train')
#parser.add_argument('--env-name', type=str, default='AirSimEnv-v42')
#parser.add_argument('--weights', type=str, default=None)
#parser.add_argument('--difficulty-level', type=str, default="default")
#args = parser.parse_args()
#args, unknown = parser.parse_known_args()
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.6
config.gpu_options.allow_growth = True
set_session(tf.Session(config=config))
# Get the environment and extract the number of actions.
#msgs.algo = "DQN"
env = gym.make(env_name)
env.init_again(eval("settings."+difficulty_level+"_range_dic"))
env.airgym.unreal_reset() #must rest so the env accomodate the changes
time.sleep(5)
np.random.seed(123)
env.seed(123)
nb_actions = env.action_space.n
WINDOW_LENGTH = 1
depth_shape = env.depth.shape
vel_shape = env.velocity.shape
dst_shape = env.position.shape
# Keras-rl interprets an extra dimension at axis=0
# added on to our observations, so we need to take it into account
img_kshape = (WINDOW_LENGTH,) + depth_shape
# Sequential model for convolutional layers applied to image
image_model = Sequential()
if(settings.policy=='deep'):
image_model.add(Conv2D(128,(3, 3), strides=(3, 3), padding='valid', activation='relu', input_shape=img_kshape,
data_format="channels_first"))
image_model.add(Conv2D(64, (3, 3), strides=(2, 2), padding='valid', activation='relu'))
image_model.add(Conv2D(32, (3, 3), strides=(1, 1), padding='valid', activation='relu'))
image_model.add(Conv2D(32, (1, 1), strides=(1, 1), padding='valid', activation='relu'))
image_model.add(Flatten())
# plot_model(image_model, to_file="model_conv_depth.png", show_shapes=True)
# Input and output of the Sequential model
image_input = Input(img_kshape)
encoded_image = image_model(image_input)
# Inputs and reshaped tensors for concatenate after with the image
velocity_input = Input((1,) + vel_shape)
distance_input = Input((1,) + dst_shape)
vel = Reshape(vel_shape)(velocity_input)
dst = Reshape(dst_shape)(distance_input)
# Concatenation of image, position, distance and geofence values.
# 3 dense layers of 256 units
denses = concatenate([encoded_image, vel, dst])
denses = Dense(1024, activation='relu')(denses)
denses = Dense(1024, activation='relu')(denses)
denses = Dense(512, activation='relu')(denses)
denses = Dense(128, activation='relu')(denses)
denses = Dense(64, activation='relu')(denses)
else:
image_model.add(Conv2D(32, (4, 4), strides=(4, 4), padding='valid', activation='relu', input_shape=img_kshape,
data_format="channels_first"))
image_model.add(Conv2D(64, (3, 3), strides=(2, 2), padding='valid', activation='relu'))
image_model.add(Conv2D(128, (2, 2), strides=(1, 1), padding='valid', activation='relu'))
image_model.add(Conv2D(64, (1, 1), strides=(1, 1), padding='valid', activation='relu'))
image_model.add(Flatten())
# plot_model(image_model, to_file="model_conv_depth.png", show_shapes=True)
# Input and output of the Sequential model
image_input = Input(img_kshape)
encoded_image = image_model(image_input)
# Inputs and reshaped tensors for concatenate after with the image
velocity_input = Input((1,) + vel_shape)
distance_input = Input((1,) + dst_shape)
vel = Reshape(vel_shape)(velocity_input)
dst = Reshape(dst_shape)(distance_input)
# Concatenation of image, position, distance and geofence values.
# 3 dense layers of 256 units
denses = concatenate([encoded_image, vel, dst])
denses = Dense(256, activation='relu')(denses)
denses = Dense(256, activation='relu')(denses)
denses = Dense(256, activation='relu')(denses)
# Last dense layer with nb_actions for the output
predictions = Dense(nb_actions, kernel_initializer='zeros', activation='linear')(denses)
model = Model(
inputs=[image_input, velocity_input, distance_input],
outputs=predictions
)
env.set_model(model)
print(model.summary())
# plot_model(model,to_file="model.png", show_shapes=True)
#train = True
#train_checkpoint = False
# Finally, we configure and compile our agent. You can use every built-in Keras optimizer and
# even the metrics!
memory = SequentialMemory(limit=100000, window_length=WINDOW_LENGTH) # reduce memmory
processor = MultiInputProcessor(nb_inputs=3)
# Select a policy. We use eps-greedy action selection, which means that a random action is selected
# with probability eps. We anneal eps from 1.0 to 0.1 over the course of 1M steps. This is done so that
# the agent initially explores the environment (high eps) and then gradually sticks to what it knows
# (low eps). We also set a dedicated eps value that is used during testing. Note that we set it to 0.05c
# so that the agent still performs some random actions. This ensures that the agent cannot get stuck.
policy = LinearAnnealedPolicy(EpsGreedyQPolicy(), attr='eps', value_max=1., value_min=.1, value_test=0.0,
nb_steps=100000)
dqn = DQNAgent(model=model, processor=processor, nb_actions=nb_actions, memory=memory, nb_steps_warmup=settings.nb_steps_warmup,
enable_double_dqn=settings.double_dqn,
enable_dueling_network=False, dueling_type='avg',
target_model_update=1e-2, policy=policy, gamma=.99)
dqn.compile(Adam(lr=0.00025), metrics=['mae'])
# Load the check-point weights and start training from there
return dqn,env
model.add(Activation('linear'))
print(model.summary())
# Finally, we configure and compile our agent. You can use every built-in Keras optimizer and
# even the metrics!
memory = SequentialMemory(limit=1000000, window_length=WINDOW_LENGTH)
processor = AtariProcessor()
# Select a policy. We use eps-greedy action selection, which means that a random action is selected
# with probability eps. We anneal eps from 1.0 to 0.1 over the course of 1M steps. This is done so that
# the agent initially explores the environment (high eps) and then gradually sticks to what it knows
# (low eps). We also set a dedicated eps value that is used during testing. Note that we set it to 0.05
# so that the agent still performs some random actions. This ensures that the agent cannot get stuck.
policy = LinearAnnealedPolicy(EpsGreedyQPolicy(),
attr='eps',
value_max=1.,
value_min=.1,
value_test=.05,
nb_steps=1000000)
# The trade-off between exploration and exploitation is difficult and an on-going research topic.
# If you want, you can experiment with the parameters or use a different policy. Another popular one
# is Boltzmann-style exploration:
# policy = BoltzmannQPolicy(tau=1.)
# Feel free to give it a try!
dqn = DQNAgent(model=model,
nb_actions=nb_actions,
policy=policy,
memory=memory,
processor=processor,
nb_steps_warmup=50000,
def training_game():
env = Environment(
map_name="HallucinIce",
visualize=True,
game_steps_per_episode=150,
agent_interface_format=features.AgentInterfaceFormat(
feature_dimensions=features.Dimensions(screen=64, minimap=32)))
input_shape = (_SIZE, _SIZE, 1)
nb_actions = 12 # Number of actions
model = neural_network_model(input_shape, nb_actions)
memory = SequentialMemory(limit=5000, window_length=_WINDOW_LENGTH)
processor = SC2Proc()
# Policy
policy = LinearAnnealedPolicy(EpsGreedyQPolicy(),
attr="eps",
value_max=1,
value_min=0.7,
value_test=.0,
nb_steps=1e6)
# Agent
dqn = DQNAgent(model=model,
nb_actions=nb_actions,
memory=memory,
enable_double_dqn=False,
nb_steps_warmup=500,
target_model_update=1e-2,
policy=policy,
batch_size=150,
processor=processor)
dqn.compile(Adam(lr=.001), metrics=["mae"])
# Save the parameters and upload them when needed
name = "HallucinIce"
w_file = "dqn_{}_weights.h5f".format(name)
check_w_file = "train_w" + name + "_weights.h5f"
if SAVE_MODEL:
check_w_file = "train_w" + name + "_weights_{step}.h5f"
log_file = "training_w_{}_log.json".format(name)
callbacks = [ModelIntervalCheckpoint(check_w_file, interval=1000)]
callbacks += [FileLogger(log_file, interval=100)]
if LOAD_MODEL:
dqn.load_weights(w_file)
dqn.fit(env,
callbacks=callbacks,
nb_steps=1e7,
action_repetition=2,
log_interval=1e4,
verbose=2)
dqn.save_weights(w_file, overwrite=True)
dqn.test(env, action_repetition=2, nb_episodes=30, visualize=False)
# Finally, we configure and compile our agent. You can use every built-in Keras optimizer and
# even the metrics!
memory = SequentialMemory(limit=1000, window_length=WINDOW_LENGTH)
processor = AtariProcessor()
# Select a policy. We use eps-greedy action selection, which means that a random action is selected
# with probability eps. We anneal eps from 1.0 to 0.1 over the course of 1M steps. This is done so that
# the agent initially explores the environment (high eps) and then gradually sticks to what it knows
# (low eps). We also set a dedicated eps value that is used during testing. Note that we set it to 0.05
# so that the agent still performs some random actions. This ensures that the agent cannot get stuck.
nb_steps = 1000
if args.weights:
nb_steps = 1
policy = LinearAnnealedPolicy(EpsGreedyQPolicy(),
attr='eps',
value_max=.5,
value_min=0,
value_test=0,
nb_steps=nb_steps)
# The trade-off between exploration and exploitation is difficult and an on-going research topic.
# If you want, you can experiment with the parameters or use a different policy. Another popular one
# is Boltzmann-style exploration:
# policy = BoltzmannQPolicy(tau=1.)
# Feel free to give it a try!
nb_steps_warmup = 100
if args.weights:
nb_steps_warmup = 100
dqn = DQNAgent(model=model,
nb_actions=nb_actions,
env.seed(123) # Next, we build a very simple model. model = NETWORK(obs_shape, nb_actions) print(model.summary()) memory = SequentialMemory( limit=MEMORY_SIZE, window_length=1 ) policy = LinearAnnealedPolicy( EpsGreedyQPolicy(), attr='eps', value_max=EPS_MAX, value_min=EPS_MIN, value_test=EPS_TEST, nb_steps=EPS_DECAY_STEPS ) dqn = DQNAgent( model=model, gamma=GAMMA, nb_actions=nb_actions, memory=memory, nb_steps_warmup=1000, target_model_update=TARGET_MODEL_UPDATE, policy=policy, test_policy=policy, enable_double_dqn=DOUBLE_DQN )