init_bias_value=0.1,
dropout=0.0,
partial_sum=1,
untie_biases=True)
l3b = cc_layers.CudaConvnetConv2DLayer(l3a,
n_filters=128,
filter_size=3,
pad=0,
weights_std=0.1,
init_bias_value=0.1,
dropout=0.0,
partial_sum=1,
untie_biases=True)
l3 = cc_layers.CudaConvnetPooling2DLayer(l3b, pool_size=2)
l3s = cc_layers.ShuffleC01BToBC01Layer(l3)
j3 = layers.MultiRotMergeLayer(l3s,
num_views=4) # 2) # merge convolutional parts
l4a = layers.DenseLayer(j3,
n_outputs=4096,
weights_std=0.001,
init_bias_value=0.01,
dropout=0.5,
nonlinearity=layers.identity)
l4b = layers.FeatureMaxPoolingLayer(l4a,
pool_size=2,
feature_dim=1,
implementation='reshape')
l4c = layers.DenseLayer(l4b,
def __init__(self,
num_actions,
phi_length,
width,
height,
discount=.9,
learning_rate=.01,
batch_size=32,
approximator='none'):
self._batch_size = batch_size
self._num_input_features = phi_length
self._phi_length = phi_length
self._img_width = width
self._img_height = height
self._discount = discount
self.num_actions = num_actions
self.learning_rate = learning_rate
self.scale_input_by = 255.0
print "neural net initialization, lr is: ", self.learning_rate, approximator
# CONSTRUCT THE LAYERS
self.q_layers = []
self.q_layers.append(
layers.Input2DLayer(self._batch_size, self._num_input_features,
self._img_height, self._img_width,
self.scale_input_by))
if approximator == 'cuda_conv':
self.q_layers.append(
cc_layers.ShuffleBC01ToC01BLayer(self.q_layers[-1]))
self.q_layers.append(
cc_layers.CudaConvnetConv2DLayer(self.q_layers[-1],
n_filters=16,
filter_size=8,
stride=4,
weights_std=.01,
init_bias_value=0.1))
self.q_layers.append(
cc_layers.CudaConvnetConv2DLayer(self.q_layers[-1],
n_filters=32,
filter_size=4,
stride=2,
weights_std=.01,
init_bias_value=0.1))
self.q_layers.append(
cc_layers.ShuffleC01BToBC01Layer(self.q_layers[-1]))
elif approximator == 'conv':
self.q_layers.append(
layers.StridedConv2DLayer(self.q_layers[-1],
n_filters=16,
filter_width=8,
filter_height=8,
stride_x=4,
stride_y=4,
weights_std=.01,
init_bias_value=0.01))
self.q_layers.append(
layers.StridedConv2DLayer(self.q_layers[-1],
n_filters=32,
filter_width=4,
filter_height=4,
stride_x=2,
stride_y=2,
weights_std=.01,
init_bias_value=0.01))
if approximator == 'cuda_conv' or approximator == 'conv':
self.q_layers.append(
layers.DenseLayer(self.q_layers[-1],
n_outputs=256,
weights_std=0.01,
init_bias_value=0.1,
dropout=0,
nonlinearity=layers.rectify))
self.q_layers.append(
layers.DenseLayer(self.q_layers[-1],
n_outputs=num_actions,
weights_std=0.01,
init_bias_value=0.1,
dropout=0,
nonlinearity=layers.identity))
if approximator == 'none':
self.q_layers.append(\
layers.DenseLayerNoBias(self.q_layers[-1],
n_outputs=num_actions,
weights_std=0.00,
dropout=0,
nonlinearity=layers.identity))
self.q_layers.append(layers.OutputLayer(self.q_layers[-1]))
for i in range(len(self.q_layers) - 1):
print self.q_layers[i].get_output_shape()
# Now create a network (using the same weights)
# for next state q values
self.next_layers = copy_layers(self.q_layers)
self.next_layers[0] = layers.Input2DLayer(self._batch_size,
self._num_input_features,
self._img_width,
self._img_height,
self.scale_input_by)
self.next_layers[1].input_layer = self.next_layers[0]
self.rewards = T.col()
self.actions = T.icol()
# Build the loss function ...
print "building loss funtion"
q_vals = self.q_layers[-1].predictions()
next_q_vals = self.next_layers[-1].predictions()
next_maxes = T.max(next_q_vals, axis=1, keepdims=True)
target = self.rewards + discount * next_maxes
target = theano.gradient.consider_constant(target)
diff = target - q_vals
# Zero out all entries for actions that were not chosen...
mask = build_mask(T.zeros_like(diff), self.actions, 1.0)
diff_masked = diff * mask
error = T.mean(diff_masked**2)
self._loss = error * diff_masked.shape[1] #
self._parameters = layers.all_parameters(self.q_layers[-1])
self._idx = T.lscalar('idx')
# CREATE VARIABLES FOR INPUT AND OUTPUT
self.states_shared = theano.shared(
np.zeros((1, 1, 1, 1), dtype=theano.config.floatX))
self.states_shared_next = theano.shared(
np.zeros((1, 1, 1, 1), dtype=theano.config.floatX))
self.rewards_shared = theano.shared(np.zeros(
(1, 1), dtype=theano.config.floatX),
broadcastable=(False, True))
self.actions_shared = theano.shared(np.zeros((1, 1), dtype='int32'),
broadcastable=(False, True))
self._givens = \
{self.q_layers[0].input_var:
self.states_shared[self._idx*self._batch_size:
(self._idx+1)*self._batch_size, :, :, :],
self.next_layers[0].input_var:
self.states_shared_next[self._idx*self._batch_size:
(self._idx+1)*self._batch_size, :, :, :],
self.rewards:
self.rewards_shared[self._idx*self._batch_size:
(self._idx+1)*self._batch_size, :],
self.actions:
self.actions_shared[self._idx*self._batch_size:
(self._idx+1)*self._batch_size, :]
}
self._updates = layers.gen_updates_rmsprop_and_nesterov_momentum(\
self._loss, self._parameters, learning_rate=self.learning_rate,
rho=0.9, momentum=0.9, epsilon=1e-6)
self._train = theano.function([self._idx],
self._loss,
givens=self._givens,
updates=self._updates)
self._compute_loss = theano.function([self._idx],
self._loss,
givens=self._givens)
self._compute_q_vals = \
theano.function([self.q_layers[0].input_var],
self.q_layers[-1].predictions(),
on_unused_input='ignore')