class DeepQNetwork:
def __init__(self, num_actions, args):
# remember parameters
self.num_actions = num_actions
self.batch_size = args.batch_size
self.discount_rate = args.discount_rate
self.history_length = args.history_length
self.screen_dim = (args.screen_height, args.screen_width)
self.clip_error = args.clip_error
# create Neon backend
self.be = gen_backend(backend=args.backend,
batch_size=args.batch_size,
rng_seed=args.random_seed,
device_id=args.device_id,
default_dtype=np.dtype(args.datatype).type,
stochastic_round=args.stochastic_round)
# prepare tensors once and reuse them
self.input_shape = (self.history_length, ) + self.screen_dim + (
self.batch_size, )
self.tensor = self.be.empty(self.input_shape)
self.tensor.lshape = self.input_shape # needed for convolutional networks
self.targets = self.be.empty((self.num_actions, self.batch_size))
# create model
layers = self.createLayers(num_actions)
self.model = Model(layers=layers)
self.cost = GeneralizedCost(costfunc=SumSquared())
self.model.initialize(self.tensor.shape[:-1], self.cost)
self.optimizer = RMSProp(learning_rate=args.learning_rate,
decay_rate=args.rmsprop_decay_rate,
stochastic_round=args.stochastic_round)
# create target model
self.target_steps = args.target_steps
self.train_iterations = 0
if self.target_steps:
self.target_model = Model(layers=self.createLayers(num_actions))
self.target_model.initialize(self.tensor.shape[:-1])
self.save_weights_path = args.save_weights_path
else:
self.target_model = self.model
self.callback = None
def createLayers(self, num_actions):
# create network
init_norm = Gaussian(loc=0.0, scale=0.01)
layers = []
# The first hidden layer convolves 32 filters of 8x8 with stride 4 with the input image and applies a rectifier nonlinearity.
layers.append(
Conv((8, 8, 32), strides=4, init=init_norm, activation=Rectlin()))
# The second hidden layer convolves 64 filters of 4x4 with stride 2, again followed by a rectifier nonlinearity.
layers.append(
Conv((4, 4, 64), strides=2, init=init_norm, activation=Rectlin()))
# This is followed by a third convolutional layer that convolves 64 filters of 3x3 with stride 1 followed by a rectifier.
layers.append(
Conv((3, 3, 64), strides=1, init=init_norm, activation=Rectlin()))
# The final hidden layer is fully-connected and consists of 512 rectifier units.
layers.append(Affine(nout=512, init=init_norm, activation=Rectlin()))
# The output layer is a fully-connected linear layer with a single output for each valid action.
layers.append(Affine(nout=num_actions, init=init_norm))
return layers
def setTensor(self, states):
# change order of axes to match what Neon expects
states = np.transpose(states, axes=(1, 2, 3, 0))
# copy() shouldn't be necessary here, but Neon doesn't work otherwise
self.tensor.set(states.copy())
# normalize network input between 0 and 1
self.be.divide(self.tensor, 255, self.tensor)
def train(self, minibatch, epoch):
# expand components of minibatch
prestates, actions, rewards, poststates, terminals = minibatch
assert len(prestates.shape) == 4
assert len(poststates.shape) == 4
assert len(actions.shape) == 1
assert len(rewards.shape) == 1
assert len(terminals.shape) == 1
assert prestates.shape == poststates.shape
assert prestates.shape[0] == actions.shape[0] == rewards.shape[
0] == poststates.shape[0] == terminals.shape[0]
if self.target_steps and self.train_iterations % self.target_steps == 0:
# HACK: push something through network, so that weights exist
self.model.fprop(self.tensor)
# HACK: serialize network to disk and read it back to clone
filename = os.path.join(self.save_weights_path,
"target_network.pkl")
save_obj(self.model.serialize(keep_states=False), filename)
self.target_model.load_weights(filename)
# feed-forward pass for poststates to get Q-values
self.setTensor(poststates)
postq = self.target_model.fprop(self.tensor, inference=True)
assert postq.shape == (self.num_actions, self.batch_size)
# calculate max Q-value for each poststate
maxpostq = self.be.max(postq, axis=0).asnumpyarray()
assert maxpostq.shape == (1, self.batch_size)
# feed-forward pass for prestates
self.setTensor(prestates)
preq = self.model.fprop(self.tensor, inference=False)
assert preq.shape == (self.num_actions, self.batch_size)
# make copy of prestate Q-values as targets
targets = preq.asnumpyarray()
# update Q-value targets for actions taken
for i, action in enumerate(actions):
if terminals[i]:
targets[action, i] = float(rewards[i])
else:
targets[action, i] = float(
rewards[i]) + self.discount_rate * maxpostq[0, i]
# copy targets to GPU memory
self.targets.set(targets)
# calculate errors
deltas = self.cost.get_errors(preq, self.targets)
assert deltas.shape == (self.num_actions, self.batch_size)
#assert np.count_nonzero(deltas.asnumpyarray()) == 32
# calculate cost, just in case
cost = self.cost.get_cost(preq, self.targets)
assert cost.shape == (1, 1)
# clip errors
if self.clip_error:
self.be.clip(deltas, -self.clip_error, self.clip_error, out=deltas)
# perform back-propagation of gradients
self.model.bprop(deltas)
# perform optimization
self.optimizer.optimize(self.model.layers_to_optimize, epoch)
# increase number of weight updates (needed for target clone interval)
self.train_iterations += 1
# calculate statistics
if self.callback:
self.callback.on_train(cost.asnumpyarray()[0, 0])
def predict(self, states):
# minibatch is full size, because Neon doesn't let change the minibatch size
assert states.shape == ((
self.batch_size,
self.history_length,
) + self.screen_dim)
# calculate Q-values for the states
self.setTensor(states)
qvalues = self.model.fprop(self.tensor, inference=True)
assert qvalues.shape == (self.num_actions, self.batch_size)
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Q-values: " + str(qvalues.asnumpyarray()[:, 0]))
# find the action with highest q-value
actions = self.be.argmax(qvalues, axis=0)
assert actions.shape == (1, self.batch_size)
# take only the first result
return actions.asnumpyarray()[0, 0]
def getMeanQ(self, states):
assert states.shape == ((
self.batch_size,
self.history_length,
) + self.screen_dim)
# calculate Q-values for the states
self.setTensor(states)
qvalues = self.model.fprop(self.tensor, inference=True)
assert qvalues.shape == (self.num_actions, self.batch_size)
# take maximum Q-value for each state
actions = self.be.max(qvalues, axis=0)
assert actions.astensor().shape == (1, self.batch_size)
# calculate mean Q-value of all states
meanq = self.be.mean(actions, axis=1)
assert meanq.astensor().shape == (1, 1)
# return the mean
return meanq.asnumpyarray()[0, 0]
def load_weights(self, load_path):
self.model.load_weights(load_path)
def save_weights(self, save_path):
save_obj(self.model.serialize(keep_states=True), save_path)
class DeepQNetwork:
def __init__(self, num_actions, args):
# create Neon backend
self.be = gen_backend(backend = args.backend,
batch_size = args.batch_size,
rng_seed = args.random_seed,
device_id = args.device_id,
default_dtype = np.dtype(args.datatype).type,
stochastic_round = args.stochastic_round)
# create model
layers = self.createLayers(num_actions)
self.model = Model(layers = layers)
self.cost = GeneralizedCost(costfunc = SumSquared())
self.optimizer = RMSProp(learning_rate = args.learning_rate,
decay_rate = args.rmsprop_decay_rate,
stochastic_round = args.stochastic_round)
# create target model
self.target_steps = args.target_steps
self.train_iterations = 0
if self.target_steps:
self.target_model = Model(layers = self.createLayers(num_actions))
self.save_weights_path = args.save_weights_path
else:
self.target_model = self.model
# remember parameters
self.num_actions = num_actions
self.batch_size = args.batch_size
self.discount_rate = args.discount_rate
self.history_length = args.history_length
self.screen_dim = (args.screen_height, args.screen_width)
self.clip_error = args.clip_error
# prepare tensors once and reuse them
self.input_shape = (self.history_length,) + self.screen_dim + (self.batch_size,)
self.tensor = self.be.empty(self.input_shape)
self.tensor.lshape = self.input_shape # needed for convolutional networks
self.targets = self.be.empty((self.num_actions, self.batch_size))
self.callback = None
def createLayers(self, num_actions):
# create network
init_norm = Gaussian(loc=0.0, scale=0.01)
layers = []
# The first hidden layer convolves 32 filters of 8x8 with stride 4 with the input image and applies a rectifier nonlinearity.
layers.append(Conv((8, 8, 32), strides=4, init=init_norm, activation=Rectlin()))
# The second hidden layer convolves 64 filters of 4x4 with stride 2, again followed by a rectifier nonlinearity.
layers.append(Conv((4, 4, 64), strides=2, init=init_norm, activation=Rectlin()))
# This is followed by a third convolutional layer that convolves 64 filters of 3x3 with stride 1 followed by a rectifier.
layers.append(Conv((3, 3, 64), strides=1, init=init_norm, activation=Rectlin()))
# The final hidden layer is fully-connected and consists of 512 rectifier units.
layers.append(Affine(nout=512, init=init_norm, activation=Rectlin()))
# The output layer is a fully-connected linear layer with a single output for each valid action.
layers.append(Affine(nout = num_actions, init = init_norm))
return layers
def setTensor(self, states):
# change order of axes to match what Neon expects
states = np.transpose(states, axes = (1, 2, 3, 0))
# copy() shouldn't be necessary here, but Neon doesn't work otherwise
self.tensor.set(states.copy())
# normalize network input between 0 and 1
self.be.divide(self.tensor, 255, self.tensor)
def train(self, minibatch, epoch):
# expand components of minibatch
prestates, actions, rewards, poststates, terminals = minibatch
assert len(prestates.shape) == 4
assert len(poststates.shape) == 4
assert len(actions.shape) == 1
assert len(rewards.shape) == 1
assert len(terminals.shape) == 1
assert prestates.shape == poststates.shape
assert prestates.shape[0] == actions.shape[0] == rewards.shape[0] == poststates.shape[0] == terminals.shape[0]
if self.target_steps and self.train_iterations % self.target_steps == 0:
# HACK: push something through network, so that weights exist
self.model.fprop(self.tensor)
# HACK: serialize network to disk and read it back to clone
filename = os.path.join(self.save_weights_path, "target_network.pkl")
save_obj(self.model.serialize(keep_states = False), filename)
self.target_model.load_weights(filename)
# feed-forward pass for poststates to get Q-values
self.setTensor(poststates)
postq = self.target_model.fprop(self.tensor, inference = True)
assert postq.shape == (self.num_actions, self.batch_size)
# calculate max Q-value for each poststate
maxpostq = self.be.max(postq, axis=0).asnumpyarray()
assert maxpostq.shape == (1, self.batch_size)
# feed-forward pass for prestates
self.setTensor(prestates)
preq = self.model.fprop(self.tensor, inference = False)
assert preq.shape == (self.num_actions, self.batch_size)
# make copy of prestate Q-values as targets
targets = preq.asnumpyarray()
# update Q-value targets for actions taken
for i, action in enumerate(actions):
if terminals[i]:
targets[action, i] = float(rewards[i])
else:
targets[action, i] = float(rewards[i]) + self.discount_rate * maxpostq[0,i]
# copy targets to GPU memory
self.targets.set(targets)
# calculate errors
deltas = self.cost.get_errors(preq, self.targets)
assert deltas.shape == (self.num_actions, self.batch_size)
#assert np.count_nonzero(deltas.asnumpyarray()) == 32
# calculate cost, just in case
cost = self.cost.get_cost(preq, self.targets)
assert cost.shape == (1,1)
# clip errors
if self.clip_error:
self.be.clip(deltas, -self.clip_error, self.clip_error, out = deltas)
# perform back-propagation of gradients
self.model.bprop(deltas)
# perform optimization
self.optimizer.optimize(self.model.layers_to_optimize, epoch)
# increase number of weight updates (needed for target clone interval)
self.train_iterations += 1
# calculate statistics
if self.callback:
self.callback.on_train(cost.asnumpyarray()[0,0])
def predict(self, states):
# minibatch is full size, because Neon doesn't let change the minibatch size
assert states.shape == ((self.batch_size, self.history_length,) + self.screen_dim)
# calculate Q-values for the states
self.setTensor(states)
qvalues = self.model.fprop(self.tensor, inference = True)
assert qvalues.shape == (self.num_actions, self.batch_size)
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Q-values: " + str(qvalues.asnumpyarray()[:,0]))
# find the action with highest q-value
actions = self.be.argmax(qvalues, axis = 0)
assert actions.shape == (1, self.batch_size)
# take only the first result
return actions.asnumpyarray()[0,0]
def getMeanQ(self, states):
assert states.shape == ((self.batch_size, self.history_length,) + self.screen_dim)
# calculate Q-values for the states
self.setTensor(states)
qvalues = self.model.fprop(self.tensor, inference = True)
assert qvalues.shape == (self.num_actions, self.batch_size)
# take maximum Q-value for each state
actions = self.be.max(qvalues, axis = 0)
assert actions.astensor().shape == (1, self.batch_size)
# calculate mean Q-value of all states
meanq = self.be.mean(actions, axis = 1)
assert meanq.astensor().shape == (1, 1)
# return the mean
return meanq.asnumpyarray()[0,0]
def load_weights(self, load_path):
self.model.load_weights(load_path)
def save_weights(self, save_path):
save_obj(self.model.serialize(keep_states = True), save_path)
class DeepQNetwork:
def __init__(self,
num_actions,
batch_size=32,
discount_rate=0.99,
history_length=4,
cols=64,
rows=64,
clip_error=1,
min_reward=-1,
max_reward=1,
batch_norm=False):
self.num_actions = num_actions
self.batch_size = batch_size
self.discount_rate = discount_rate
self.history_length = history_length
self.board_dim = (cols, rows)
self.clip_error = clip_error
self.min_reward = min_reward
self.max_reward = max_reward
self.batch_norm = batch_norm
self.be = gen_backend(backend='gpu',
batch_size=self.batch_size,
datatype=np.dtype('float32').type)
self.input_shape = (self.history_length, ) + self.board_dim + (
self.batch_size, )
self.input = self.be.empty(self.input_shape)
self.input.lshape = self.input_shape # hack from simple_dqn "needed for convolutional networks"
self.targets = self.be.empty((self.num_actions, self.batch_size))
layers = self._createLayers(self.num_actions)
self.model = Model(layers=layers)
self.cost = GeneralizedCost(costfunc=SumSquared())
# for l in self.model.layers.layers:
# l.parallelism = 'Disabled'
self.model.initialize(self.input_shape[:-1], cost=self.cost)
self.optimizer = RMSProp(learning_rate=0.002,
decay_rate=0.95,
stochastic_round=True)
self.train_iterations = 0
self.target_model = Model(layers=self._createLayers(num_actions))
# for l in self.target_model.layers.layers:
# l.parallelism = 'Disabled'
self.target_model.initialize(self.input_shape[:-1])
self.callback = None
def _createLayers(self, num_actions):
init_xavier_conv = Xavier(local=True)
init_xavier_affine = Xavier(local=False)
layers = []
layers.append(
Conv((8, 8, 32),
strides=4,
init=init_xavier_conv,
activation=Rectlin(),
batch_norm=self.batch_norm))
layers.append(
Conv((4, 4, 64),
strides=2,
init=init_xavier_conv,
activation=Rectlin(),
batch_norm=self.batch_norm))
layers.append(
Conv((2, 2, 128),
strides=1,
init=init_xavier_conv,
activation=Rectlin(),
batch_norm=self.batch_norm))
layers.append(
Affine(nout=256,
init=init_xavier_affine,
activation=Rectlin(),
batch_norm=self.batch_norm))
layers.append(Affine(nout=num_actions, init=init_xavier_affine))
return layers
def _setInput(self, states):
states = np.transpose(states, axes=(1, 2, 3, 0))
self.input.set(states.copy())
self.be.add(self.input, 1, self.input)
self.be.divide(self.input, 2, self.input)
def update_target_network(self):
pdict = self.model.get_description(get_weights=True, keep_states=True)
self.target_model.deserialize(pdict, load_states=True)
def train(self, minibatch, epoch):
prestates, actions, rewards, poststates, terminals = minibatch
self._setInput(poststates)
postq = self.target_model.fprop(self.input, inference=True)
assert postq.shape == (self.num_actions, self.batch_size)
maxpostq = self.be.max(postq, axis=0).asnumpyarray()
assert maxpostq.shape == (1, self.batch_size)
self._setInput(prestates)
preq = self.model.fprop(self.input, inference=False)
assert preq.shape == (self.num_actions, self.batch_size)
targets = preq.asnumpyarray().copy()
rewards = np.clip(rewards, -1, 1)
for i, action in enumerate(actions):
if terminals[i]:
targets[action, i] = float(rewards[i])
else:
targets[action, i] = float(
rewards[i]) + self.discount_rate * maxpostq[0, i]
self.targets.set(targets)
deltas = self.cost.get_errors(preq, self.targets)
assert deltas.shape == (self.num_actions, self.batch_size)
cost = self.cost.get_cost(preq, self.targets)
assert cost.shape == (1, 1)
if self.clip_error:
self.be.clip(deltas, -self.clip_error, self.clip_error, out=deltas)
self.model.bprop(deltas)
self.optimizer.optimize(self.model.layers_to_optimize, epoch)
self.train_iterations += 1
self.callback.on_train(cost[0, 0])
def predict(self, states):
assert states.shape == ((
self.batch_size,
self.history_length,
) + self.board_dim)
self._setInput(states)
qvalues = self.model.fprop(self.input, inference=True)
assert qvalues.shape == (self.num_actions, self.batch_size)
return qvalues.T.asnumpyarray()
def load_weights(self, load_path):
self.model.load_params(load_path)
def save_weights(self, save_path):
self.model.save_params(save_path)
class ModelRunnerNeon():
def __init__(self, args, max_action_no, batch_dimension):
self.args = args
self.train_batch_size = args.train_batch_size
self.discount_factor = args.discount_factor
self.use_gpu_replay_mem = args.use_gpu_replay_mem
self.be = gen_backend(backend='gpu', batch_size=self.train_batch_size)
self.input_shape = (batch_dimension[1], batch_dimension[2],
batch_dimension[3], batch_dimension[0])
self.input = self.be.empty(self.input_shape)
self.input.lshape = self.input_shape # HACK: needed for convolutional networks
self.targets = self.be.empty((max_action_no, self.train_batch_size))
if self.use_gpu_replay_mem:
self.history_buffer = self.be.zeros(batch_dimension,
dtype=np.uint8)
self.input_uint8 = self.be.empty(self.input_shape, dtype=np.uint8)
else:
self.history_buffer = np.zeros(batch_dimension, dtype=np.float32)
self.train_net = Model(self.create_layers(max_action_no))
self.cost = GeneralizedCost(costfunc=SumSquared())
# Bug fix
for l in self.train_net.layers.layers:
l.parallelism = 'Disabled'
self.train_net.initialize(self.input_shape[:-1], self.cost)
self.target_net = Model(self.create_layers(max_action_no))
# Bug fix
for l in self.target_net.layers.layers:
l.parallelism = 'Disabled'
self.target_net.initialize(self.input_shape[:-1])
if self.args.optimizer == 'Adam': # Adam
self.optimizer = Adam(beta_1=args.rms_decay,
beta_2=args.rms_decay,
learning_rate=args.learning_rate)
else: # Neon RMSProp
self.optimizer = RMSProp(decay_rate=args.rms_decay,
learning_rate=args.learning_rate)
self.max_action_no = max_action_no
self.running = True
def get_initializer(self, input_size):
dnnInit = self.args.dnn_initializer
if dnnInit == 'xavier':
initializer = Xavier()
elif dnnInit == 'fan_in':
std_dev = 1.0 / math.sqrt(input_size)
initializer = Uniform(low=-std_dev, high=std_dev)
else:
initializer = Gaussian(0, 0.01)
return initializer
def create_layers(self, max_action_no):
layers = []
initializer = self.get_initializer(input_size=4 * 8 * 8)
layers.append(
Conv(fshape=(8, 8, 32),
strides=4,
init=initializer,
bias=initializer,
activation=Rectlin()))
initializer = self.get_initializer(input_size=32 * 4 * 4)
layers.append(
Conv(fshape=(4, 4, 64),
strides=2,
init=initializer,
bias=initializer,
activation=Rectlin()))
initializer = self.get_initializer(input_size=64 * 3 * 3)
layers.append(
Conv(fshape=(3, 3, 64),
strides=1,
init=initializer,
bias=initializer,
activation=Rectlin()))
initializer = self.get_initializer(input_size=7 * 7 * 64)
layers.append(
Affine(nout=512,
init=initializer,
bias=initializer,
activation=Rectlin()))
initializer = self.get_initializer(input_size=512)
layers.append(
Affine(nout=max_action_no, init=initializer, bias=initializer))
return layers
def clip_reward(self, reward):
if reward > self.args.clip_reward_high:
return self.args.clip_reward_high
elif reward < self.args.clip_reward_low:
return self.args.clip_reward_low
else:
return reward
def set_input(self, data):
if self.use_gpu_replay_mem:
self.be.copy_transpose(data, self.input_uint8, axes=(1, 2, 3, 0))
self.input[:] = self.input_uint8 / 255
else:
self.input.set(data.transpose(1, 2, 3, 0).copy())
self.be.divide(self.input, 255, self.input)
def predict(self, history_buffer):
self.set_input(history_buffer)
output = self.train_net.fprop(self.input, inference=True)
return output.T.asnumpyarray()[0]
def print_weights(self):
pass
def train(self, minibatch, replay_memory, learning_rate, debug):
if self.args.prioritized_replay == True:
prestates, actions, rewards, poststates, terminals, replay_indexes, heap_indexes, weights = minibatch
else:
prestates, actions, rewards, poststates, terminals = minibatch
# Get Q*(s, a) with targetNet
self.set_input(poststates)
post_qvalue = self.target_net.fprop(self.input,
inference=True).T.asnumpyarray()
if self.args.double_dqn == True:
# Get Q*(s, a) with trainNet
post_qvalue2 = self.train_net.fprop(
self.input, inference=True).T.asnumpyarray()
# Get Q(s, a) with trainNet
self.set_input(prestates)
pre_qvalue = self.train_net.fprop(self.input, inference=False)
label = pre_qvalue.asnumpyarray().copy()
for i in range(0, self.train_batch_size):
if self.args.clip_reward:
reward = self.clip_reward(rewards[i])
else:
reward = rewards[i]
if terminals[i]:
label[actions[i], i] = reward
else:
if self.args.double_dqn == True:
max_index = np.argmax(post_qvalue2[i])
label[actions[i],
i] = reward + self.discount_factor * post_qvalue[i][
max_index]
else:
label[actions[i],
i] = reward + self.discount_factor * np.max(
post_qvalue[i])
# copy targets to GPU memory
self.targets.set(label)
delta = self.cost.get_errors(pre_qvalue, self.targets)
if self.args.prioritized_replay == True:
delta_value = delta.asnumpyarray()
for i in range(self.train_batch_size):
if debug:
print 'weight[%s]: %.5f, delta: %.5f, newDelta: %.5f' % (
i, weights[i], delta_value[actions[i], i],
weights[i] * delta_value[actions[i], i])
replay_memory.update_td(heap_indexes[i],
abs(delta_value[actions[i], i]))
delta_value[actions[i],
i] = weights[i] * delta_value[actions[i], i]
delta.set(delta_value.copy())
if self.args.clip_loss:
self.be.clip(delta, -1.0, 1.0, out=delta)
self.train_net.bprop(delta)
self.optimizer.optimize(self.train_net.layers_to_optimize, epoch=0)
def update_model(self):
# have to serialize also states for batch normalization to work
pdict = self.train_net.get_description(get_weights=True,
keep_states=True)
self.target_net.deserialize(pdict, load_states=True)
#print ('Updated target model')
def finish_train(self):
self.running = False
def load(self, file_name):
self.train_net.load_params(file_name)
self.update_model()
def save(self, file_name):
self.train_net.save_params(file_name)
class ModelRunnerNeon():
def __init__(self, args, max_action_no, batch_dimension):
self.args = args
self.train_batch_size = args.train_batch_size
self.discount_factor = args.discount_factor
self.use_gpu_replay_mem = args.use_gpu_replay_mem
self.be = gen_backend(backend='gpu',
batch_size=self.train_batch_size)
self.input_shape = (batch_dimension[1], batch_dimension[2], batch_dimension[3], batch_dimension[0])
self.input = self.be.empty(self.input_shape)
self.input.lshape = self.input_shape # HACK: needed for convolutional networks
self.targets = self.be.empty((max_action_no, self.train_batch_size))
if self.use_gpu_replay_mem:
self.history_buffer = self.be.zeros(batch_dimension, dtype=np.uint8)
self.input_uint8 = self.be.empty(self.input_shape, dtype=np.uint8)
else:
self.history_buffer = np.zeros(batch_dimension, dtype=np.float32)
self.train_net = Model(self.create_layers(max_action_no))
self.cost = GeneralizedCost(costfunc=SumSquared())
# Bug fix
for l in self.train_net.layers.layers:
l.parallelism = 'Disabled'
self.train_net.initialize(self.input_shape[:-1], self.cost)
self.target_net = Model(self.create_layers(max_action_no))
# Bug fix
for l in self.target_net.layers.layers:
l.parallelism = 'Disabled'
self.target_net.initialize(self.input_shape[:-1])
if self.args.optimizer == 'Adam': # Adam
self.optimizer = Adam(beta_1=args.rms_decay,
beta_2=args.rms_decay,
learning_rate=args.learning_rate)
else: # Neon RMSProp
self.optimizer = RMSProp(decay_rate=args.rms_decay,
learning_rate=args.learning_rate)
self.max_action_no = max_action_no
self.running = True
def get_initializer(self, input_size):
dnnInit = self.args.dnn_initializer
if dnnInit == 'xavier':
initializer = Xavier()
elif dnnInit == 'fan_in':
std_dev = 1.0 / math.sqrt(input_size)
initializer = Uniform(low=-std_dev, high=std_dev)
else:
initializer = Gaussian(0, 0.01)
return initializer
def create_layers(self, max_action_no):
layers = []
initializer = self.get_initializer(input_size = 4 * 8 * 8)
layers.append(Conv(fshape=(8, 8, 32), strides=4, init=initializer, bias=initializer, activation=Rectlin()))
initializer = self.get_initializer(input_size = 32 * 4 * 4)
layers.append(Conv(fshape=(4, 4, 64), strides=2, init=initializer, bias=initializer, activation=Rectlin()))
initializer = self.get_initializer(input_size = 64 * 3 * 3)
layers.append(Conv(fshape=(3, 3, 64), strides=1, init=initializer, bias=initializer, activation=Rectlin()))
initializer = self.get_initializer(input_size = 7 * 7 * 64)
layers.append(Affine(nout=512, init=initializer, bias=initializer, activation=Rectlin()))
initializer = self.get_initializer(input_size = 512)
layers.append(Affine(nout=max_action_no, init=initializer, bias=initializer))
return layers
def clip_reward(self, reward):
if reward > self.args.clip_reward_high:
return self.args.clip_reward_high
elif reward < self.args.clip_reward_low:
return self.args.clip_reward_low
else:
return reward
def set_input(self, data):
if self.use_gpu_replay_mem:
self.be.copy_transpose(data, self.input_uint8, axes=(1, 2, 3, 0))
self.input[:] = self.input_uint8 / 255
else:
self.input.set(data.transpose(1, 2, 3, 0).copy())
self.be.divide(self.input, 255, self.input)
def predict(self, history_buffer):
self.set_input(history_buffer)
output = self.train_net.fprop(self.input, inference=True)
return output.T.asnumpyarray()[0]
def print_weights(self):
pass
def train(self, minibatch, replay_memory, learning_rate, debug):
if self.args.prioritized_replay == True:
prestates, actions, rewards, poststates, terminals, replay_indexes, heap_indexes, weights = minibatch
else:
prestates, actions, rewards, poststates, terminals = minibatch
# Get Q*(s, a) with targetNet
self.set_input(poststates)
post_qvalue = self.target_net.fprop(self.input, inference=True).T.asnumpyarray()
if self.args.double_dqn == True:
# Get Q*(s, a) with trainNet
post_qvalue2 = self.train_net.fprop(self.input, inference=True).T.asnumpyarray()
# Get Q(s, a) with trainNet
self.set_input(prestates)
pre_qvalue = self.train_net.fprop(self.input, inference=False)
label = pre_qvalue.asnumpyarray().copy()
for i in range(0, self.train_batch_size):
if self.args.clip_reward:
reward = self.clip_reward(rewards[i])
else:
reward = rewards[i]
if terminals[i]:
label[actions[i], i] = reward
else:
if self.args.double_dqn == True:
max_index = np.argmax(post_qvalue2[i])
label[actions[i], i] = reward + self.discount_factor* post_qvalue[i][max_index]
else:
label[actions[i], i] = reward + self.discount_factor* np.max(post_qvalue[i])
# copy targets to GPU memory
self.targets.set(label)
delta = self.cost.get_errors(pre_qvalue, self.targets)
if self.args.prioritized_replay == True:
delta_value = delta.asnumpyarray()
for i in range(self.train_batch_size):
if debug:
print 'weight[%s]: %.5f, delta: %.5f, newDelta: %.5f' % (i, weights[i], delta_value[actions[i], i], weights[i] * delta_value[actions[i], i])
replay_memory.update_td(heap_indexes[i], abs(delta_value[actions[i], i]))
delta_value[actions[i], i] = weights[i] * delta_value[actions[i], i]
delta.set(delta_value.copy())
if self.args.clip_loss:
self.be.clip(delta, -1.0, 1.0, out = delta)
self.train_net.bprop(delta)
self.optimizer.optimize(self.train_net.layers_to_optimize, epoch=0)
def update_model(self):
# have to serialize also states for batch normalization to work
pdict = self.train_net.get_description(get_weights=True, keep_states=True)
self.target_net.deserialize(pdict, load_states=True)
#print ('Updated target model')
def finish_train(self):
self.running = False
def load(self, file_name):
self.train_net.load_params(file_name)
self.update_model()
def save(self, file_name):
self.train_net.save_params(file_name)