class CriticCNNPolicy: def __init__(self, name, observation_space_shape, num_actions, pretrained_policy=None, *args, **kwargs): self.name = name self.observation_space_shape = observation_space_shape self.num_actions = num_actions self._build_network(pretrained_policy) self.trainer = Trainer(self.value, self.loss, [ adam(self.value.parameters, lr=[0.0001] * 600 + [0.00005] * 600 + [0.00001] * 600 + [0.000005], momentum=0.9) ]) def _build_network(self, pretrained_policy): self.image_frame = C.input_variable((1, ) + self.observation_space_shape) self.target_current_state_value = C.input_variable((1, )) if pretrained_policy is None: h = C.layers.Convolution2D(filter_shape=(7, 7), num_filters=32, strides=(4, 4), pad=True, name='conv_1', activation=C.relu)(self.image_frame) h = C.layers.Convolution2D(filter_shape=(5, 5), num_filters=64, strides=(2, 2), pad=True, name='conv_2', activation=C.relu)(h) h = C.layers.Convolution2D(filter_shape=(3, 3), num_filters=128, strides=(1, 1), pad=True, name='conv_3', activation=C.relu)(h) h = C.layers.Dense(64, activation=C.relu, name='dense_1')(h) self.value = C.layers.Dense(1, name='dense_2')(h) else: self.value = C.Function.load(pretrained_policy)(self.image_frame) self.loss = C.squared_error(self.target_current_state_value, self.value) def optimise(self, image_frame, target_current_state_value): self.trainer.train_minibatch({ self.image_frame: image_frame, self.target_current_state_value: target_current_state_value }) def predict(self, image_frame): return self.value.eval({self.image_frame: image_frame})
class StackedFrameCNNPolicy: def __init__(self, name, num_frames_to_stack, observation_space_shape, num_actions, pretrained_policy=None, *args, **kwargs): self.name = name self.num_frames_to_stack = num_frames_to_stack self.observation_space_shape = observation_space_shape self.frame_stacker = FrameStacker(stack_size=num_frames_to_stack, frame_shape=observation_space_shape) self.num_actions = num_actions self._build_network(pretrained_policy) self.trainer = Trainer(self.q, self.loss, [sgd(self.q.parameters, lr=0.000001)]) def _build_network(self, pretrained_policy): self.image_frame = C.input_variable((self.num_frames_to_stack, ) + self.observation_space_shape) if pretrained_policy is None: h = C.layers.Convolution2D(filter_shape=(7, 7), num_filters=32, strides=(4, 4), pad=True, name='conv_1', activation=C.relu)(self.image_frame) h = C.layers.Convolution2D(filter_shape=(5, 5), num_filters=64, strides=(2, 2), pad=True, name='conv_2', activation=C.relu)(h) h = C.layers.Convolution2D(filter_shape=(3, 3), num_filters=128, strides=(1, 1), pad=True, name='conv_3', activation=C.relu)(h) h = C.layers.Dense(64, activation=C.relu, name='dense_1')(h) self.q = C.layers.Dense(self.num_actions, name='dense_1')(h) else: self.q = C.Function.load(pretrained_policy)(self.image_frame) self.q_target = C.input_variable(self.num_actions) self.loss = C.mean(C.losses.squared_error(self.q_target, self.q)) def optimise(self, image_frame, q_target): self.trainer.train_minibatch({ self.image_frame: image_frame, self.q_target: q_target }) def predict(self, image_frame): return self.q.eval({self.image_frame: image_frame})
def __init__(self, name, observation_space_shape, num_actions, pretrained_policy=None, *args, **kwargs): self.name = name self.observation_space_shape = observation_space_shape self.num_actions = num_actions self._build_network(pretrained_policy) self.trainer = Trainer(self.q, self.loss, [sgd(self.q.parameters, lr=5e-4)])
def __init__(self, name, observation_space_shape, num_actions, pretrained_policy=None, *args, **kwargs): self.name = name self.observation_space_shape = observation_space_shape self.num_actions = num_actions self._build_network(pretrained_policy) self.trainer = Trainer( self.value, self.loss, [adam(self.value.parameters, lr=0.001, momentum=0.9)])
class ActorNNPolicy: def __init__(self, name, observation_space_shape, num_actions, pretrained_policy=None, *args, **kwargs): self.name = name self.observation_space_shape = observation_space_shape self.num_actions = num_actions self._build_network(pretrained_policy) self.trainer = Trainer( self.probabilities, self.loss, [adam(self.probabilities.parameters, lr=0.0001, momentum=0.9)]) def _build_network(self, pretrained_policy): self.image_frame = C.input_variable((1, ) + self.observation_space_shape) self.td_error = C.input_variable((1, )) self.action_index = C.input_variable((1, )) one_hot_action = C.one_hot(self.action_index, self.num_actions) if pretrained_policy is None: h = C.layers.Dense(64, activation=C.relu, name='dense_1')(self.image_frame) self.probabilities = C.layers.Dense(self.num_actions, name='dense_2', activation=C.softmax)(h) else: self.probabilities = C.Function.load(pretrained_policy)( self.image_frame) selected_action_probablity = C.ops.times_transpose( self.probabilities, one_hot_action) self.log_probability = C.ops.log(selected_action_probablity) self.loss = -self.td_error * self.log_probability # self.probabilities = C.softmax(self.logits) # log_probability_of_action_taken = cross_entropy_with_softmax(self.logits, one_hot_action) # self.loss = C.reduce_mean(self.td_error*log_probability_of_action_taken) def optimise(self, image_frame, td_error, action_index): self.trainer.train_minibatch({ self.image_frame: image_frame, self.td_error: td_error, self.action_index: action_index }) def predict(self, image_frame): return self.probabilities.eval({self.image_frame: image_frame})
def __init__(self, name, num_frames_to_stack, observation_space_shape, num_actions, pretrained_policy=None, *args, **kwargs): self.name = name self.num_frames_to_stack = num_frames_to_stack self.observation_space_shape = observation_space_shape self.num_actions = num_actions self._build_network(pretrained_policy) self.trainer = Trainer( self.log_probability, self.loss, [adam(self.probabilities.parameters, lr=0.00001, momentum=0.9)])
class REINFORCENNPolicy: def __init__(self, name, observation_space_shape, num_actions, pretrained_policy=None, *args, **kwargs): self.name = name self.observation_space_shape = observation_space_shape self.num_actions = num_actions self._build_network(pretrained_policy) self.trainer = Trainer( self.action_probabilities, self.loss, [sgd(self.action_probabilities.parameters, lr=0.000001)]) def _build_network(self, pretrained_policy): self.input = C.input_variable(self.observation_space_shape, name='image frame') self.target = C.input_variable((1, ), name='q_target') self.action_index = C.input_variable((1, )) one_hot_action = C.ops.squeeze( C.one_hot(self.action_index, self.num_actions)) if pretrained_policy is None: h = C.layers.Dense(64, activation=C.relu, name='dense_1')(self.input) h = C.layers.Dense(32, activation=C.relu, name='dense_1')(h) self.action_probabilities = C.layers.Dense(self.num_actions, activation=C.softmax, name='dense_1')(h) else: self.action_probabilities = C.Function.load(pretrained_policy)( self.input) selected_action_probablity = C.ops.times_transpose( self.action_probabilities, one_hot_action) self.log_probability = C.ops.log(selected_action_probablity) self.loss = C.sum(self.log_probability * self.target) def optimise(self, state, action, target): self.trainer.train_minibatch({ self.input: state, self.action_index: action, self.target: target }) def predict(self, state): return self.action_probabilities.eval({self.input: state})
def __init__(self, name, num_frames_to_stack, observation_space_shape, num_actions, pretrained_policy=None, *args, **kwargs): self.name = name self.num_frames_to_stack = num_frames_to_stack self.observation_space_shape = observation_space_shape self.frame_stacker = FrameStacker(stack_size=num_frames_to_stack, frame_shape=observation_space_shape) self.num_actions = num_actions self._build_network(pretrained_policy) self.trainer = Trainer(self.q, self.loss, [sgd(self.q.parameters, lr=0.000001)])
class ActorNNPolicy: def __init__(self, name, observation_space_shape, num_actions, pretrained_policy=None, *args, **kwargs): self.name = name self.observation_space_shape = observation_space_shape self.num_actions = num_actions self._build_network(pretrained_policy) self.trainer = Trainer( self.log_probability, self.loss, [adam(self.probabilities.parameters, lr=0.001, momentum=0.9)]) def _build_network(self, pretrained_policy): self.input = C.input_variable(self.observation_space_shape) self.td_error = C.input_variable((1, )) self.action_index = C.input_variable((1, )) one_hot_action = C.ops.squeeze( C.one_hot(self.action_index, self.num_actions)) if pretrained_policy is None: h = C.layers.Dense(64, activation=C.relu, name='dense_1')(self.input) h = C.layers.Dense(64, activation=C.tanh, name='dense_2')(h) self.probabilities = C.layers.Dense(self.num_actions, name='dense_3', activation=C.softmax)(h) else: self.probabilities = C.Function.load(pretrained_policy)(self.input) selected_action_probablity = C.ops.times_transpose( self.probabilities, one_hot_action) self.log_probability = C.ops.log(selected_action_probablity) self.loss = -self.td_error * self.log_probability def optimise(self, state, td_error, action_index): self.trainer.train_minibatch({ self.input: state, self.td_error: td_error, self.action_index: action_index }) def predict(self, state): return self.probabilities.eval({self.input: state})
class CriticNNPolicy: def __init__(self, name, observation_space_shape, num_actions, pretrained_policy=None, *args, **kwargs): self.name = name self.observation_space_shape = observation_space_shape self.num_actions = num_actions self._build_network(pretrained_policy) self.trainer = Trainer( self.value, self.loss, [adam(self.value.parameters, lr=0.001, momentum=0.9)]) def _build_network(self, pretrained_policy): self.input = C.input_variable(self.observation_space_shape) self.target_current_state_value = C.input_variable((1, )) if pretrained_policy is None: h = C.layers.Dense(64, activation=C.relu, name='dense_1')(self.input) h = C.layers.Dense(64, activation=C.relu, name='dense_2')(h) self.value = C.layers.Dense(1, name='dense_3')(h) else: self.value = C.Function.load(pretrained_policy)(self.input) self.loss = C.squared_error(self.target_current_state_value, self.value) def optimise(self, state, target_current_state_value): self.trainer.train_minibatch({ self.input: state, self.target_current_state_value: target_current_state_value }) def predict(self, state): return self.value.eval({self.input: state})
class SimpleNNPolicy: def __init__(self, name, observation_space_shape, num_actions, pretrained_policy=None, *args, **kwargs): self.name = name self.observation_space_shape = observation_space_shape self.num_actions = num_actions self._build_network(pretrained_policy) self.trainer = Trainer(self.q, self.loss, [sgd(self.q.parameters, lr=5e-4)]) def _build_network(self, pretrained_policy): self.input = C.input_variable(self.observation_space_shape, name='image frame') if pretrained_policy is None: h = C.layers.Dense(64, activation=C.relu, name='dense_1')(self.input) h = C.layers.Dense(64, activation=C.relu, name='dense_2')(h) self.q = C.layers.Dense(self.num_actions, activation=None, name='output')(h) else: self.q = C.Function.load(pretrained_policy)(self.input) self.q_target = C.input_variable(self.num_actions, name='q_target') self.loss = C.losses.squared_error(self.q_target, self.q) def optimise(self, state, q_target): self.trainer.train_minibatch({ self.input: state, self.q_target: q_target }) def predict(self, state): return self.q.eval({self.input: state})
def __init__(self, in_shape, output_shape, device_id=None, learning_rate=0.00025, momentum=0.9, minibatch_size=32, update_interval=10000, n_workers=1, visualizer=None): """ Q Neural Network following Mnih and al. implementation and default options. The network has the following topology: Convolution(32, (8, 8)) Convolution(64, (4, 4)) Convolution(64, (2, 2)) Dense(512) :param in_shape: Shape of the observations perceived by the learner (the neural net input) :param output_shape: Size of the action space (mapped to the number of output neurons) :param device_id: Use None to let CNTK select the best available device, -1 for CPU, >= 0 for GPU (default: None) :param learning_rate: Learning rate (default: 0.00025, as per Mnih et al.) :param momentum: Momentum, provided as momentum value for averaging gradients without unit gain filter Note that CNTK does not currently provide an implementation of Graves' RmsProp with momentum. It uses AdamSGD optimizer instead. (default: 0, no momentum with RProp optimizer) :param minibatch_size: Minibatch size (default: 32, as per Mnih et al.) :param n_workers: Number of concurrent worker for distributed training. (default: 1, not distributed) :param visualizer: Optional visualizer allowing the model to save summary data (default: None, no visualization) Ref: Mnih et al.: "Human-level control through deep reinforcement learning." Nature 518.7540 (2015): 529-533. """ assert learning_rate > 0, 'learning_rate should be > 0' assert 0. <= momentum < 1, 'momentum should be 0 <= momentum < 1' QModel.__init__(self, in_shape, output_shape) CntkModel.__init__(self, device_id, False, n_workers, visualizer) self._nb_actions = output_shape self._steps = 0 self._target_update_interval = update_interval self._target = None # Input vars self._environment = input(in_shape, name='env', dynamic_axes=(Axis.default_batch_axis())) self._q_targets = input(1, name='q_targets', dynamic_axes=(Axis.default_batch_axis())) self._actions = input(output_shape, name='actions', dynamic_axes=(Axis.default_batch_axis())) # Define the neural network graph self._model = self._build_model()(self._environment) self._target = self._model.clone( CloneMethod.freeze, {self._environment: self._environment} ) # Define the learning rate lr_schedule = learning_rate_schedule(learning_rate, UnitType.minibatch) # AdamSGD optimizer m_schedule = momentum_schedule(momentum) vm_schedule = momentum_schedule(0.999) l_sgd = adam(self._model.parameters, lr_schedule, momentum=m_schedule, unit_gain=True, variance_momentum=vm_schedule) if self.distributed_training: raise NotImplementedError('ASGD not implemented yet.') # _actions is a sparse 1-hot encoding of the actions done by the agent q_acted = reduce_sum(self._model * self._actions, axis=0) # Define the trainer with Huber Loss function criterion = huber_loss(q_acted, self._q_targets, 1.0) self._learner = l_sgd self._trainer = Trainer(self._model, (criterion, None), l_sgd)
class QNeuralNetwork(CntkModel, QModel): """ Represents a learning capable entity using CNTK """ def __init__(self, in_shape, output_shape, device_id=None, learning_rate=0.00025, momentum=0.9, minibatch_size=32, update_interval=10000, n_workers=1, visualizer=None): """ Q Neural Network following Mnih and al. implementation and default options. The network has the following topology: Convolution(32, (8, 8)) Convolution(64, (4, 4)) Convolution(64, (2, 2)) Dense(512) :param in_shape: Shape of the observations perceived by the learner (the neural net input) :param output_shape: Size of the action space (mapped to the number of output neurons) :param device_id: Use None to let CNTK select the best available device, -1 for CPU, >= 0 for GPU (default: None) :param learning_rate: Learning rate (default: 0.00025, as per Mnih et al.) :param momentum: Momentum, provided as momentum value for averaging gradients without unit gain filter Note that CNTK does not currently provide an implementation of Graves' RmsProp with momentum. It uses AdamSGD optimizer instead. (default: 0, no momentum with RProp optimizer) :param minibatch_size: Minibatch size (default: 32, as per Mnih et al.) :param n_workers: Number of concurrent worker for distributed training. (default: 1, not distributed) :param visualizer: Optional visualizer allowing the model to save summary data (default: None, no visualization) Ref: Mnih et al.: "Human-level control through deep reinforcement learning." Nature 518.7540 (2015): 529-533. """ assert learning_rate > 0, 'learning_rate should be > 0' assert 0. <= momentum < 1, 'momentum should be 0 <= momentum < 1' QModel.__init__(self, in_shape, output_shape) CntkModel.__init__(self, device_id, False, n_workers, visualizer) self._nb_actions = output_shape self._steps = 0 self._target_update_interval = update_interval self._target = None # Input vars self._environment = input(in_shape, name='env', dynamic_axes=(Axis.default_batch_axis())) self._q_targets = input(1, name='q_targets', dynamic_axes=(Axis.default_batch_axis())) self._actions = input(output_shape, name='actions', dynamic_axes=(Axis.default_batch_axis())) # Define the neural network graph self._model = self._build_model()(self._environment) self._target = self._model.clone( CloneMethod.freeze, {self._environment: self._environment}) # Define the learning rate lr_schedule = learning_rate_schedule(learning_rate, UnitType.minibatch) # AdamSGD optimizer m_schedule = momentum_schedule(momentum) vm_schedule = momentum_schedule(0.999) l_sgd = adam(self._model.parameters, lr_schedule, momentum=m_schedule, unit_gain=True, variance_momentum=vm_schedule) if self.distributed_training: raise NotImplementedError('ASGD not implemented yet.') # _actions is a sparse 1-hot encoding of the actions done by the agent q_acted = reduce_sum(self._model * self._actions, axis=0) # Define the trainer with Huber Loss function criterion = huber_loss(q_acted, self._q_targets, 1.0) self._learner = l_sgd self._trainer = Trainer(self._model, (criterion, None), l_sgd) @property def loss_val(self): return self._trainer.previous_minibatch_loss_average def _build_model(self): with default_options(init=he_uniform(), activation=relu, bias=True): model = Sequential([ Convolution((8, 8), 32, strides=(4, 4)), Convolution((4, 4), 64, strides=(2, 2)), Convolution((3, 3), 64, strides=(1, 1)), Dense(512, init=he_normal(0.01)), Dense(self._nb_actions, activation=None, init=he_normal(0.01)) ]) return model def train(self, x, q_value_targets, actions=None): assert actions is not None, 'actions cannot be None' # We need to add extra dimensions to shape [N, 1] => [N, 1] if check_rank(q_value_targets.shape, 1): q_value_targets = q_value_targets.reshape((-1, 1)) # Add extra dimensions to match shape [N, 1] required by one_hot if check_rank(actions.shape, 1): actions = actions.reshape((-1, 1)) # We need batch axis if check_rank(x.shape, len(self._environment.shape)): x = prepend_batch_axis(x) self._trainer.train_minibatch({ self._environment: x, self._actions: Value.one_hot(actions, self._nb_actions), self._q_targets: q_value_targets }) # Counter number of train calls self._steps += 1 # Update the model with the target one if (self._steps % self._target_update_interval) == 0: self._target = self._model.clone( CloneMethod.freeze, {self._environment: self._environment}) def evaluate(self, data, model=QModel.ACTION_VALUE_NETWORK): # If evaluating a single sample, expand the minibatch axis # (minibatch = 1, input_shape...) if len(data.shape) == len(self.input_shape): data = prepend_batch_axis(data) # Append minibatch dim if model == QModel.TARGET_NETWORK: predictions = self._target.eval({self._environment: data}) else: predictions = self._model.eval({self._environment: data}) return predictions.squeeze()
class ActorStackedFrameCNNPolicy: def __init__(self, name, num_frames_to_stack, observation_space_shape, num_actions, pretrained_policy=None, *args, **kwargs): self.name = name self.num_frames_to_stack = num_frames_to_stack self.observation_space_shape = observation_space_shape self.num_actions = num_actions self._build_network(pretrained_policy) self.trainer = Trainer( self.log_probability, self.loss, [adam(self.probabilities.parameters, lr=0.00001, momentum=0.9)]) def _build_network(self, pretrained_policy): self.image_frame = C.input_variable((self.num_frames_to_stack, ) + self.observation_space_shape) self.td_error = C.input_variable((1, )) self.action_index = C.input_variable((1, )) one_hot_action = C.ops.squeeze( C.one_hot(self.action_index, self.num_actions)) if pretrained_policy is None: h = C.layers.Convolution2D(filter_shape=(7, 7), num_filters=32, strides=(4, 4), pad=True, name='conv_1', activation=C.relu)(self.image_frame) h = C.layers.Convolution2D(filter_shape=(5, 5), num_filters=64, strides=(2, 2), pad=True, name='conv_2', activation=C.relu)(h) h = C.layers.Convolution2D(filter_shape=(3, 3), num_filters=128, strides=(1, 1), pad=True, name='conv_3', activation=C.relu)(h) h = C.layers.Dense(64, activation=C.relu, name='dense_1')(h) self.probabilities = C.layers.Dense(self.num_actions, name='dense_2', activation=C.softmax)(h) else: self.probabilities = C.Function.load(pretrained_policy)( self.image_frame) selected_action_probablity = C.ops.times_transpose( self.probabilities, one_hot_action) self.log_probability = C.ops.log(selected_action_probablity) self.loss = -self.td_error * self.log_probability def optimise(self, image_frame, td_error, action_index): self.trainer.train_minibatch({ self.image_frame: image_frame, self.td_error: td_error, self.action_index: action_index }) def predict(self, image_frame): return self.probabilities.eval({self.image_frame: image_frame})
class CriticCNNPolicy: def __init__(self, name, observation_space_shape, num_actions, pretrained_policy=None, *args, **kwargs): self.name = name self.observation_space_shape = observation_space_shape self.num_actions = num_actions self._build_network(pretrained_policy) self.trainer = Trainer( self.value, self.loss, [adam(self.value.parameters, lr=0.0001, momentum=0.9)]) def _build_network(self, pretrained_policy): self.image_frame = C.input_variable((1, ) + self.observation_space_shape) self.next_image_frame = C.input_variable((1, ) + self.observation_space_shape) self.reward = C.input_variable((1, )) if pretrained_policy is None: h = C.layers.Convolution2D(filter_shape=(7, 7), num_filters=32, strides=(4, 4), pad=True, name='conv_1', activation=C.relu) h = C.layers.Convolution2D(filter_shape=(5, 5), num_filters=64, strides=(2, 2), pad=True, name='conv_2', activation=C.relu)(h) h = C.layers.Convolution2D(filter_shape=(3, 3), num_filters=128, strides=(1, 1), pad=True, name='conv_3', activation=C.relu)(h) h = C.layers.Dense(64, activation=C.relu, name='dense_1')(h) v = C.layers.Dense(1, name='dense_2')(h) self.value = v(self.image_frame) self.next_value = v(self.next_image_frame) self.output = C.combine([self.value, self.next_value]) else: self.output = C.Function.load(pretrained_policy)( self.image_frame, self.next_image_frame) [self.value, self.next_value] = self.output[ self.value.output], self.output[self.next_value.output] target = DISCOUNT_FACTOR * self.next_value + self.reward self.loss = C.squared_error(target, self.value) def optimise(self, image_frame, next_image_frame, rewards): self.trainer.train_minibatch({ self.image_frame: image_frame, self.next_image_frame: next_image_frame, self.reward: rewards }) def predict(self, image_frame): return self.value.eval({self.image_frame: image_frame})
class ActorCriticCNNPolicy: def __init__(self, name, observation_space_shape, num_actions, pretrained_policy=None, *args, **kwargs): self.name = name self.observation_space_shape = observation_space_shape self.num_actions = num_actions self._build_network(pretrained_policy) self.trainer = Trainer( self.output, self.loss, [adam(self.output.parameters, lr=0.00003, momentum=0.9)]) def _build_network(self, pretrained_policy): self.image_frame = C.input_variable((1, ) + self.observation_space_shape) self.next_image_frame = C.input_variable((1, ) + self.observation_space_shape) self.advantage = C.input_variable((1, )) self.action_index = C.input_variable((1, )) self.target_value = C.input_variable((1, )) one_hot_action = C.one_hot(self.action_index, self.num_actions) if pretrained_policy is None: h = C.layers.Convolution2D(filter_shape=(7, 7), num_filters=32, strides=(4, 4), pad=True, name='conv_1', activation=C.relu) h = C.layers.Convolution2D(filter_shape=(5, 5), num_filters=64, strides=(2, 2), pad=True, name='conv_2', activation=C.relu)(h) h = C.layers.Convolution2D(filter_shape=(3, 3), num_filters=128, strides=(1, 1), pad=True, name='conv_3', activation=C.relu)(h) h = C.layers.Dense(64, activation=C.relu, name='dense_1')(h) self.probabilities = C.layers.Dense(self.num_actions, name='dense_2', activation=C.softmax)(h( self.image_frame)) v = C.layers.Dense(1, name='dense_3')(h) self.value = v(self.image_frame) self.next_value = v(self.next_image_frame) self.output = C.combine( [self.probabilities, self.value, self.next_value]) else: [self.probabilities, self.value, self.next_value] = list( C.Function.load(pretrained_policy)(self.image_frame, self.next_image_frame)) self.values_output = C.combine([self.value, self.next_value]) selected_action_probablity = C.ops.times_transpose( self.probabilities, one_hot_action) self.log_probability = C.ops.log(selected_action_probablity) self.actor_loss = -self.advantage * self.log_probability self.critic_loss = C.squared_error(self.target_value, self.value) self.loss = 0.5 * self.actor_loss + 0.5 * self.critic_loss # self.probabilities = C.softmax(self.logits) # log_probability_of_action_taken = cross_entropy_with_softmax(self.logits, one_hot_action) # self.loss = C.reduce_mean(self.td_error*log_probability_of_action_taken) def optimise(self, image_frame, action_index, advantage, targets): self.trainer.train_minibatch({ self.image_frame: image_frame, self.advantage: advantage, self.action_index: action_index, self.target_value: targets }) def predict(self, image_frame): return self.probabilities.eval({self.image_frame: image_frame}) def values(self, image_frame, next_image_frame): output = self.values_output.eval({ self.image_frame: image_frame, self.next_image_frame: next_image_frame }) return output[self.value.output], output[self.next_value.output]
classificationError = classification_error(outputLayer, labelsShape) input_map = { featuresShape: reader.streams.features, labelsShape: reader.streams.labels } numOfEpochs = 10 printer = [ProgressPrinter( tag = 'Training', num_epochs = numOfEpochs)] learningRate = learning_rate_schedule([0.1, 0.01, 0.001], UnitType.sample, 700) trainer = Trainer(outputLayer,(crossEntropy, classificationError), [adadelta(outputLayer.parameters, learningRate)], printer) minibatchSize = 50 numberOfSamples = 2208 numberOfSweepsForTraining = 10 trainingSession = training_session( trainer=trainer, mb_source=reader, mb_size=minibatchSize, model_inputs_to_streams=input_map, max_samples=numberOfSamples * numberOfSweepsForTraining, progress_frequency=numberOfSamples ) trainingSession.train()
classificationError = classification_error(outputLayer, labelsShape) input_map = { featuresShape: reader.streams.features, labelsShape: reader.streams.labels } numOfEpochs = 10 printer = [ProgressPrinter( tag = 'Training', num_epochs = numOfEpochs)] learningRate = learning_rate_schedule([0.1, 0.01, 0.001], UnitType.sample, 700) trainer = Trainer(outputLayer,(crossEntropy, classificationError), [adadelta(outputLayer.parameters, learningRate)], printer) minibatchSize = 50 numberOfSamples = 2208 numberOfSweepsForTraining = 10 trainingSession = training_session( trainer=trainer, mb_source=reader, mb_size=minibatchSize, model_inputs_to_streams=input_map, max_samples=numberOfSamples * numberOfSweepsForTraining, progress_frequency=numberOfSamples ) trainingSession.train()
def __init__(self, in_shape, output_shape, device_id=None, learning_rate=0.00025, momentum=0.9, minibatch_size=32, update_interval=10000, n_workers=1, visualizer=None): """ Q Neural Network following Mnih and al. implementation and default options. The network has the following topology: Convolution(32, (8, 8)) Convolution(64, (4, 4)) Convolution(64, (2, 2)) Dense(512) :param in_shape: Shape of the observations perceived by the learner (the neural net input) :param output_shape: Size of the action space (mapped to the number of output neurons) :param device_id: Use None to let CNTK select the best available device, -1 for CPU, >= 0 for GPU (default: None) :param learning_rate: Learning rate (default: 0.00025, as per Mnih et al.) :param momentum: Momentum, provided as momentum value for averaging gradients without unit gain filter Note that CNTK does not currently provide an implementation of Graves' RmsProp with momentum. It uses AdamSGD optimizer instead. (default: 0, no momentum with RProp optimizer) :param minibatch_size: Minibatch size (default: 32, as per Mnih et al.) :param n_workers: Number of concurrent worker for distributed training. (default: 1, not distributed) :param visualizer: Optional visualizer allowing the model to save summary data (default: None, no visualization) Ref: Mnih et al.: "Human-level control through deep reinforcement learning." Nature 518.7540 (2015): 529-533. """ assert learning_rate > 0, 'learning_rate should be > 0' assert 0. <= momentum < 1, 'momentum should be 0 <= momentum < 1' QModel.__init__(self, in_shape, output_shape) CntkModel.__init__(self, device_id, False, n_workers, visualizer) self._nb_actions = output_shape self._steps = 0 self._target_update_interval = update_interval self._target = None # Input vars self._environment = input(in_shape, name='env', dynamic_axes=(Axis.default_batch_axis())) self._q_targets = input(1, name='q_targets', dynamic_axes=(Axis.default_batch_axis())) self._actions = input(output_shape, name='actions', dynamic_axes=(Axis.default_batch_axis())) # Define the neural network graph self._model = self._build_model()(self._environment) self._target = self._model.clone( CloneMethod.freeze, {self._environment: self._environment}) # Define the learning rate lr_schedule = learning_rate_schedule(learning_rate, UnitType.minibatch) # AdamSGD optimizer m_schedule = momentum_schedule(momentum) vm_schedule = momentum_schedule(0.999) l_sgd = adam(self._model.parameters, lr_schedule, momentum=m_schedule, unit_gain=True, variance_momentum=vm_schedule) if self.distributed_training: raise NotImplementedError('ASGD not implemented yet.') # _actions is a sparse 1-hot encoding of the actions done by the agent q_acted = reduce_sum(self._model * self._actions, axis=0) # Define the trainer with Huber Loss function criterion = huber_loss(q_acted, self._q_targets, 1.0) self._learner = l_sgd self._trainer = Trainer(self._model, (criterion, None), l_sgd)
class QNeuralNetwork(CntkModel, QModel): """ Represents a learning capable entity using CNTK """ def __init__(self, in_shape, output_shape, device_id=None, learning_rate=0.00025, momentum=0.9, minibatch_size=32, update_interval=10000, n_workers=1, visualizer=None): """ Q Neural Network following Mnih and al. implementation and default options. The network has the following topology: Convolution(32, (8, 8)) Convolution(64, (4, 4)) Convolution(64, (2, 2)) Dense(512) :param in_shape: Shape of the observations perceived by the learner (the neural net input) :param output_shape: Size of the action space (mapped to the number of output neurons) :param device_id: Use None to let CNTK select the best available device, -1 for CPU, >= 0 for GPU (default: None) :param learning_rate: Learning rate (default: 0.00025, as per Mnih et al.) :param momentum: Momentum, provided as momentum value for averaging gradients without unit gain filter Note that CNTK does not currently provide an implementation of Graves' RmsProp with momentum. It uses AdamSGD optimizer instead. (default: 0, no momentum with RProp optimizer) :param minibatch_size: Minibatch size (default: 32, as per Mnih et al.) :param n_workers: Number of concurrent worker for distributed training. (default: 1, not distributed) :param visualizer: Optional visualizer allowing the model to save summary data (default: None, no visualization) Ref: Mnih et al.: "Human-level control through deep reinforcement learning." Nature 518.7540 (2015): 529-533. """ assert learning_rate > 0, 'learning_rate should be > 0' assert 0. <= momentum < 1, 'momentum should be 0 <= momentum < 1' QModel.__init__(self, in_shape, output_shape) CntkModel.__init__(self, device_id, False, n_workers, visualizer) self._nb_actions = output_shape self._steps = 0 self._target_update_interval = update_interval self._target = None # Input vars self._environment = input(in_shape, name='env', dynamic_axes=(Axis.default_batch_axis())) self._q_targets = input(1, name='q_targets', dynamic_axes=(Axis.default_batch_axis())) self._actions = input(output_shape, name='actions', dynamic_axes=(Axis.default_batch_axis())) # Define the neural network graph self._model = self._build_model()(self._environment) self._target = self._model.clone( CloneMethod.freeze, {self._environment: self._environment} ) # Define the learning rate lr_schedule = learning_rate_schedule(learning_rate, UnitType.minibatch) # AdamSGD optimizer m_schedule = momentum_schedule(momentum) vm_schedule = momentum_schedule(0.999) l_sgd = adam(self._model.parameters, lr_schedule, momentum=m_schedule, unit_gain=True, variance_momentum=vm_schedule) if self.distributed_training: raise NotImplementedError('ASGD not implemented yet.') # _actions is a sparse 1-hot encoding of the actions done by the agent q_acted = reduce_sum(self._model * self._actions, axis=0) # Define the trainer with Huber Loss function criterion = huber_loss(q_acted, self._q_targets, 1.0) self._learner = l_sgd self._trainer = Trainer(self._model, (criterion, None), l_sgd) @property def loss_val(self): return self._trainer.previous_minibatch_loss_average def _build_model(self): with default_options(init=he_uniform(), activation=relu, bias=True): model = Sequential([ Convolution((8, 8), 32, strides=(4, 4)), Convolution((4, 4), 64, strides=(2, 2)), Convolution((3, 3), 64, strides=(1, 1)), Dense(512, init=he_normal(0.01)), Dense(self._nb_actions, activation=None, init=he_normal(0.01)) ]) return model def train(self, x, q_value_targets, actions=None): assert actions is not None, 'actions cannot be None' # We need to add extra dimensions to shape [N, 1] => [N, 1] if check_rank(q_value_targets.shape, 1): q_value_targets = q_value_targets.reshape((-1, 1)) # Add extra dimensions to match shape [N, 1] required by one_hot if check_rank(actions.shape, 1): actions = actions.reshape((-1, 1)) # We need batch axis if check_rank(x.shape, len(self._environment.shape)): x = prepend_batch_axis(x) self._trainer.train_minibatch({ self._environment: x, self._actions: Value.one_hot(actions, self._nb_actions), self._q_targets: q_value_targets }) # Counter number of train calls self._steps += 1 # Update the model with the target one if (self._steps % self._target_update_interval) == 0: self._target = self._model.clone( CloneMethod.freeze, {self._environment: self._environment} ) def evaluate(self, data, model=QModel.ACTION_VALUE_NETWORK): # If evaluating a single sample, expand the minibatch axis # (minibatch = 1, input_shape...) if len(data.shape) == len(self.input_shape): data = prepend_batch_axis(data) # Append minibatch dim if model == QModel.TARGET_NETWORK: predictions = self._target.eval({self._environment: data}) else: predictions = self._model.eval({self._environment: data}) return predictions.squeeze()