def __init__(self, name, input_shape, extra_input_shape, output_dim, hidden_sizes, conv_filters, conv_filter_sizes, conv_strides, conv_pads, extra_hidden_sizes=None, hidden_W_init=L.XavierUniformInitializer(), hidden_b_init=tf.zeros_initializer, output_W_init=L.XavierUniformInitializer(), output_b_init=tf.zeros_initializer, hidden_nonlinearity=tf.nn.relu, output_nonlinearity=None, input_var=None, input_layer=None): Serializable.quick_init(self, locals()) if extra_hidden_sizes is None: extra_hidden_sizes = [] with tf.variable_scope(name): input_flat_dim = np.prod(input_shape) extra_input_flat_dim = np.prod(extra_input_shape) total_input_flat_dim = input_flat_dim + extra_input_flat_dim if input_layer is None: l_in = L.InputLayer(shape=(None, total_input_flat_dim), input_var=input_var, name="input") else: l_in = input_layer l_conv_in = L.reshape(L.SliceLayer(l_in, indices=slice(input_flat_dim), name="conv_slice"), ([0], ) + input_shape, name="conv_reshaped") l_extra_in = L.reshape(L.SliceLayer(l_in, indices=slice( input_flat_dim, None), name="extra_slice"), ([0], ) + extra_input_shape, name="extra_reshaped") l_conv_hid = l_conv_in for idx, conv_filter, filter_size, stride, pad in zip( range(len(conv_filters)), conv_filters, conv_filter_sizes, conv_strides, conv_pads, ): l_conv_hid = L.Conv2DLayer( l_conv_hid, num_filters=conv_filter, filter_size=filter_size, stride=(stride, stride), pad=pad, nonlinearity=hidden_nonlinearity, name="conv_hidden_%d" % idx, ) l_extra_hid = l_extra_in for idx, hidden_size in enumerate(extra_hidden_sizes): l_extra_hid = L.DenseLayer( l_extra_hid, num_units=hidden_size, nonlinearity=hidden_nonlinearity, name="extra_hidden_%d" % idx, W=hidden_W_init, b=hidden_b_init, ) l_joint_hid = L.concat( [L.flatten(l_conv_hid, name="conv_hidden_flat"), l_extra_hid], name="joint_hidden") for idx, hidden_size in enumerate(hidden_sizes): l_joint_hid = L.DenseLayer( l_joint_hid, num_units=hidden_size, nonlinearity=hidden_nonlinearity, name="joint_hidden_%d" % idx, W=hidden_W_init, b=hidden_b_init, ) l_out = L.DenseLayer( l_joint_hid, num_units=output_dim, nonlinearity=output_nonlinearity, name="output", W=output_W_init, b=output_b_init, ) self._l_in = l_in self._l_out = l_out LayersPowered.__init__(self, [l_out], input_layers=[l_in])
def __init__(self, name, input_shape, output_dim, conv_filters, conv_filter_sizes, conv_strides, conv_pads, hidden_sizes, hidden_nonlinearity, output_nonlinearity, hidden_W_init=L.XavierUniformInitializer(), hidden_b_init=tf.zeros_initializer, output_W_init=L.XavierUniformInitializer(), output_b_init=tf.zeros_initializer, input_var=None, input_layer=None, batch_normalization=False, weight_normalization=False): Serializable.quick_init(self, locals()) """ A network composed of several convolution layers followed by some fc layers. input_shape: (width,height,channel) HOWEVER, network inputs are assumed flattened. This network will first unflatten the inputs and then apply the standard convolutions and so on. conv_filters: a list of numbers of convolution kernel conv_filter_sizes: a list of sizes (int) of the convolution kernels conv_strides: a list of strides (int) of the conv kernels conv_pads: a list of pad formats (either 'SAME' or 'VALID') hidden_nonlinearity: a nonlinearity from tf.nn, shared by all conv and fc layers hidden_sizes: a list of numbers of hidden units for all fc layers """ with tf.variable_scope(name, reuse=tf.AUTO_REUSE): if input_layer is not None: l_in = input_layer l_hid = l_in elif len(input_shape) == 3: l_in = L.InputLayer(shape=(None, np.prod(input_shape)), input_var=input_var, name="input") l_hid = L.reshape(l_in, ([0], ) + input_shape, name="reshape_input") elif len(input_shape) == 2: l_in = L.InputLayer(shape=(None, np.prod(input_shape)), input_var=input_var, name="input") input_shape = (1, ) + input_shape l_hid = L.reshape(l_in, ([0], ) + input_shape, name="reshape_input") else: l_in = L.InputLayer(shape=(None, ) + input_shape, input_var=input_var, name="input") l_hid = l_in if batch_normalization: l_hid = L.batch_norm(l_hid) for idx, conv_filter, filter_size, stride, pad in zip( range(len(conv_filters)), conv_filters, conv_filter_sizes, conv_strides, conv_pads, ): # print("debug123",conv_filter,filter_size,stride,pad,hidden_nonlinearity,idx,weight_normalization) l_hid = L.Conv2DLayer( l_hid, num_filters=conv_filter, filter_size=filter_size, stride=(stride, stride), pad=pad, nonlinearity=hidden_nonlinearity, name="conv_hidden_%d" % idx, weight_normalization=weight_normalization, ) if batch_normalization: l_hid = L.batch_norm(l_hid) if output_nonlinearity == L.spatial_expected_softmax: assert len(hidden_sizes) == 0 assert output_dim == conv_filters[-1] * 2 l_hid.nonlinearity = tf.identity l_out = L.SpatialExpectedSoftmaxLayer(l_hid) else: l_hid = L.flatten(l_hid, name="conv_flatten") for idx, hidden_size in enumerate(hidden_sizes): l_hid = L.DenseLayer( l_hid, num_units=hidden_size, nonlinearity=hidden_nonlinearity, name="hidden_%d" % idx, W=hidden_W_init, b=hidden_b_init, weight_normalization=weight_normalization, ) if batch_normalization: l_hid = L.batch_norm(l_hid) l_out = L.DenseLayer( l_hid, num_units=output_dim, nonlinearity=output_nonlinearity, name="output", W=output_W_init, b=output_b_init, weight_normalization=weight_normalization, ) if batch_normalization: l_out = L.batch_norm(l_out) self._l_in = l_in self._l_out = l_out # self._input_var = l_in.input_var LayersPowered.__init__(self, l_out)
def __init__( self, name, output_dim, output_dim_binary, hidden_sizes, hidden_nonlinearity, output_nonlinearity, output_nonlinearity_binary, hidden_W_init=L.XavierUniformInitializer(), hidden_b_init=tf.zeros_initializer(), output_W_init=L.XavierUniformInitializer(), output_b_init=tf.zeros_initializer(), input_var=None, input_layer=None, input_shape=None, batch_normalization=False, weight_normalization=False, ): Serializable.quick_init(self, locals()) with tf.variable_scope(name): if input_layer is None: l_in = L.InputLayer(shape=(None, ) + input_shape, input_var=input_var, name="input") else: l_in = input_layer self._layers = [l_in] l_hid = l_in if batch_normalization: l_hid = L.batch_norm(l_hid) for idx, hidden_size in enumerate(hidden_sizes): l_hid = L.DenseLayer(l_hid, num_units=hidden_size, nonlinearity=hidden_nonlinearity, name="hidden_%d" % idx, W=hidden_W_init, b=hidden_b_init, weight_normalization=weight_normalization) if batch_normalization: l_hid = L.batch_norm(l_hid) self._layers.append(l_hid) l_hid_binary = L.DenseLayer( l_hid, num_units=hidden_size, nonlinearity=hidden_nonlinearity, name="hidden_binary", W=hidden_W_init, b=hidden_b_init, weight_normalization=weight_normalization) l_out_binary = L.DenseLayer( l_hid_binary, num_units=output_dim_binary, nonlinearity=output_nonlinearity_binary, name="output_binary", W=output_W_init, b=output_b_init, weight_normalization=weight_normalization) self._layers.append(l_out_binary) l_hid_out = L.DenseLayer(l_hid, num_units=hidden_size, nonlinearity=hidden_nonlinearity, name="hidden_final", W=hidden_W_init, b=hidden_b_init, weight_normalization=weight_normalization) l_out = L.DenseLayer(l_hid_out, num_units=output_dim, nonlinearity=output_nonlinearity, name="output", W=output_W_init, b=output_b_init, weight_normalization=weight_normalization) if batch_normalization: l_out = L.batch_norm(l_out) self._layers.append(l_out) self._l_in = l_in self._l_out = l_out self._l_out_binary = l_out_binary self._output_binary = L.get_output(l_out_binary) self._output = L.get_output(l_out) LayersPowered.__init__(self, [l_out, l_out_binary])
def __init__( self, name, output_dim, hidden_sizes, hidden_nonlinearity, hidden_W_init=L.XavierUniformInitializer(), hidden_b_init=tf.zeros_initializer(), output_W_init=L.XavierUniformInitializer(), output_b_init=tf.zeros_initializer(), input_var=None, input_layer=None, input_shape=None, batch_normalization=False, weight_normalization=False, ): Serializable.quick_init(self, locals()) with tf.variable_scope(name): if input_layer is None: assert input_shape is not None, \ "input_layer or input_shape must be supplied" l_in = L.InputLayer(shape=(None, ) + input_shape, input_var=input_var, name="input") else: l_in = input_layer self._layers = [l_in] l_hid = l_in if batch_normalization: l_hid = L.batch_norm(l_hid) for idx, hidden_size in enumerate(hidden_sizes): l_hid = L.DenseLayer(l_hid, num_units=hidden_size, nonlinearity=hidden_nonlinearity, name="hidden_%d" % idx, W=hidden_W_init, b=hidden_b_init, weight_normalization=weight_normalization) if batch_normalization: l_hid = L.batch_norm(l_hid) self._layers.append(l_hid) l_out_raw = L.DenseLayer(l_hid, num_units=output_dim, name="output", W=output_W_init, b=output_b_init, weight_normalization=weight_normalization) if batch_normalization: l_out_raw = L.batch_norm(l_out_raw) self._layers.append(l_out_raw) # mask assumed to occupy first output_dim elements def mask_op(X): return X[..., :output_dim] def mask_shape_op(old_shape): return old_shape[:-1] + (output_dim, ) mask = L.OpLayer(l_in, mask_op, shape_op=mask_shape_op) self._layers.append(mask) l_out = L.OpLayer(l_out_raw, masked_softmax, extras=[mask]) self._layers.append(l_out) self._l_in = l_in self._l_out = l_out # self._input_var = l_in.input_var self._output = L.get_output(l_out) LayersPowered.__init__(self, l_out)
def _make_weights(self, old_prop_weights=None, old_act_weights=None): # prop_weights[i] is a dictionary mapping predicate names to weights # for modules in the i-th proposition layer self.prop_weights = [] self.act_weights = [] self.all_weights = [] for hid_idx, hid_sizes in enumerate(self.hidden_sizes): act_size, prop_size = hid_sizes # make action layer weights act_dict = {} for unbound_act in self.dom_meta.unbound_acts: preds = self.dom_meta.rel_pred_names(unbound_act) if not hid_idx: # first layer, so our input is actually a binary vector # giving a truth value for each proposition in_size = len(preds) * 2 + self.extra_dim else: in_size = len(preds) * self.hidden_sizes[hid_idx - 1][1] name_pfx = 'hid_%d_act_%s' % (hid_idx, unbound_act.schema_name) # TODO: doing "if old_act_weights" check each time is silly. # Should store parameters *purely* by name, and have code # responsible for automatically re-instantiating old weights if # they exist. if old_act_weights is not None: W_init, b_init = map(L.const, old_act_weights[hid_idx][unbound_act]) else: W_init = L.XavierUniformInitializer() b_init = tf.zeros_initializer() act_W = L.create_param( W_init, shape=(1, in_size, act_size), name=name_pfx + '/W') act_b = L.create_param( b_init, shape=(act_size, ), name=name_pfx + '/b') act_dict[unbound_act] = (act_W, act_b) self.all_weights.extend([act_W, act_b]) self.act_weights.append(act_dict) # make hidden proposition layer weights pred_dict = {} for pred_name in self.dom_meta.pred_names: rel_acts = self.dom_meta.rel_acts(pred_name) # We should never end up with NO relevant actions for a # predicate. Why bother including the predicate? assert len(rel_acts) > 0, \ "no relevant actions for proposition %s" % pred_name in_size = len(rel_acts) * act_size name_pfx = 'hid_%d_prop_%s' % (hid_idx, pred_name) if old_prop_weights is not None: W_init, b_init = map(L.const, old_prop_weights[hid_idx][pred_name]) else: W_init = L.XavierUniformInitializer() b_init = tf.zeros_initializer() prop_W = L.create_param( W_init, shape=(1, in_size, prop_size), name=name_pfx + '/W') prop_b = L.create_param( b_init, shape=(prop_size, ), name=name_pfx + '/b') pred_dict[pred_name] = (prop_W, prop_b) self.all_weights.extend([prop_W, prop_b]) self.prop_weights.append(pred_dict) # make final layer weights (action) final_act_dict = {} for unbound_act in self.dom_meta.unbound_acts: preds = self.dom_meta.rel_pred_names(unbound_act) if not self.hidden_sizes: in_size = len(preds) * 2 + self.extra_dim else: in_size = len(preds) * self.hidden_sizes[-1][1] name_pfx = 'final_act_%s' % unbound_act.schema_name if old_act_weights is not None: W_init, b_init = map(L.const, old_act_weights[-1][unbound_act]) else: W_init = L.XavierUniformInitializer() b_init = tf.zeros_initializer() final_act_W = L.create_param( W_init, shape=(1, in_size, 1), name=name_pfx + '/W') final_act_b = L.create_param( b_init, shape=(1, ), name=name_pfx + '/b') final_act_dict[unbound_act] = (final_act_W, final_act_b) self.all_weights.extend([final_act_W, final_act_b]) self.act_weights.append(final_act_dict)
def __init__(self, env_spec, name='qnet', hidden_sizes=(32, 32), hidden_nonlinearity=tf.nn.relu, action_merge_layer=-2, output_nonlinearity=None, hidden_W_init=L.XavierUniformInitializer(), hidden_b_init=tf.zeros_initializer(), output_W_init=L.XavierUniformInitializer(), output_b_init=tf.zeros_initializer(), bn=False): Serializable.quick_init(self, locals()) with tf.variable_scope(name): l_obs = L.InputLayer(shape=(None, env_spec.observation_space.flat_dim), name="obs") l_action = L.InputLayer(shape=(None, env_spec.action_space.flat_dim), name="actions") n_layers = len(hidden_sizes) + 1 if n_layers > 1: action_merge_layer = \ (action_merge_layer % n_layers + n_layers) % n_layers else: action_merge_layer = 1 l_hidden = l_obs for idx, size in enumerate(hidden_sizes): if bn: l_hidden = L.batch_norm(l_hidden) if idx == action_merge_layer: l_hidden = L.ConcatLayer([l_hidden, l_action]) l_hidden = L.DenseLayer(l_hidden, num_units=size, W=hidden_W_init, b=hidden_b_init, nonlinearity=hidden_nonlinearity, name="h%d" % (idx + 1)) if action_merge_layer == n_layers: l_hidden = L.ConcatLayer([l_hidden, l_action]) l_output = L.DenseLayer(l_hidden, num_units=1, W=output_W_init, b=output_b_init, nonlinearity=output_nonlinearity, name="output") #output_var = L.get_output(l_output, deterministic=True).flatten() output_var = tf.reshape(L.get_output(l_output, deterministic=True), (-1, )) self._f_qval = tensor_utils.compile_function( [l_obs.input_var, l_action.input_var], output_var) self._output_layer = l_output self._obs_layer = l_obs self._action_layer = l_action self._output_nonlinearity = output_nonlinearity LayersPowered.__init__(self, [l_output])
def __init__( self, name, output_dim, hidden_sizes, hidden_nonlinearity, output_nonlinearity, hidden_W_init=L.XavierUniformInitializer(), hidden_b_init=tf.zeros_initializer(), output_W_init=L.XavierUniformInitializer(), output_b_init=tf.zeros_initializer(), input_var=None, input_layer=None, input_shape=None, batch_normalization=False, weight_normalization=False, # added arguments w_auxiliary=False, auxliary_classes=0., ): Serializable.quick_init(self, locals()) with tf.variable_scope(name): if input_layer is None: l_in = L.InputLayer(shape=(None, ) + input_shape, input_var=input_var, name="input") else: l_in = input_layer self._layers = [l_in] l_hid = l_in if batch_normalization: l_hid = L.batch_norm(l_hid) for idx, hidden_size in enumerate(hidden_sizes): l_hid = L.DenseLayer(l_hid, num_units=hidden_size, nonlinearity=hidden_nonlinearity, name="hidden_%d" % idx, W=hidden_W_init, b=hidden_b_init, weight_normalization=weight_normalization) if batch_normalization: l_hid = L.batch_norm(l_hid) self._layers.append(l_hid) if w_auxiliary: assert auxliary_classes > 0 l_hid_aux = L.DenseLayer( l_hid, num_units=64, nonlinearity=hidden_nonlinearity, name="auxiliary_hidden_0", W=hidden_W_init, b=hidden_b_init, weight_normalization=weight_normalization) if batch_normalization: l_hid_aux = L.batch_norm(l_hid_aux) self._layers.append(l_hid_aux) l_aux = L.DenseLayer(l_hid_aux, num_units=auxliary_classes, nonlinearity=output_nonlinearity, name="aux_output", W=output_W_init, b=output_b_init, weight_normalization=weight_normalization) if batch_normalization: l_aux = L.batch_norm(l_aux) self._layers.append(l_aux) self._l_aux = l_aux l_out = L.DenseLayer(l_hid, num_units=output_dim, nonlinearity=output_nonlinearity, name="output", W=output_W_init, b=output_b_init, weight_normalization=weight_normalization) if batch_normalization: l_out = L.batch_norm(l_out) self._layers.append(l_out) self._l_in = l_in self._l_out = l_out # self._input_var = l_in.input_var self._output = L.get_output(l_out) LayersPowered.__init__(self, l_out)
def __init__(self, env_spec, name='nafqnet', hidden_sizes=(32, 32), hidden_nonlinearity=tf.nn.relu, action_merge_layer=0, output_nonlinearity=None, hidden_W_init=L.XavierUniformInitializer(), hidden_b_init=L.ZerosInitializer(), output_W_init=L.XavierUniformInitializer(), output_b_init=L.ZerosInitializer(), bn=False): Serializable.quick_init(self, locals()) assert not env_spec.action_space.is_discrete action_dim = env_spec.action_space.flat_dim self._action_dim = action_dim self._env_spec = env_spec n_layers = len(hidden_sizes) action_merge_layer = \ (action_merge_layer % n_layers + n_layers) % n_layers with tf.variable_scope(name): l_obs = L.InputLayer(shape=(None, env_spec.observation_space.flat_dim), name="obs") l_action = L.InputLayer(shape=(None, env_spec.action_space.flat_dim), name="actions") l_policy_mu = L.InputLayer(shape=(None, action_dim), name="policy_mu") l_policy_sigma = L.InputLayer(shape=(None, action_dim, action_dim), name="policy_sigma") l_hidden = l_obs idx = 0 l_hidden_kwargs = dict( W=hidden_W_init, b=hidden_b_init, nonlinearity=hidden_nonlinearity, ) l_output_kwargs = dict( W=output_W_init, b=output_b_init, ) while idx < action_merge_layer: if bn: l_hidden = L.batch_norm(l_hidden) l_hidden = L.DenseLayer( l_hidden, num_units=hidden_sizes[idx], name="h%d" % (idx + 1), **l_hidden_kwargs, ) idx += 1 _idx = idx _l_hidden = l_hidden # compute L network while idx < n_layers: if bn: l_hidden = L.batch_norm(l_hidden) l_hidden = L.DenseLayer( l_hidden, num_units=hidden_sizes[idx], name="L_h%d" % (idx + 1), **l_hidden_kwargs, ) idx += 1 l_L = L.DenseLayer( l_hidden, num_units=action_dim**2, nonlinearity=None, name="L_h%d" % (idx + 1), **l_output_kwargs, ) # compute V network idx = _idx l_hidden = _l_hidden while idx < n_layers: if bn: l_hidden = L.batch_norm(l_hidden) l_hidden = L.DenseLayer( l_hidden, num_units=hidden_sizes[idx], name="V_h%d" % (idx + 1), **l_hidden_kwargs, ) idx += 1 l_V = L.DenseLayer( l_hidden, num_units=1, nonlinearity=None, name="V_h%d" % (idx + 1), **l_output_kwargs, ) # compute mu network idx = _idx l_hidden = _l_hidden while idx < n_layers: if bn: l_hidden = L.batch_norm(l_hidden) l_hidden = L.DenseLayer( l_hidden, num_units=hidden_sizes[idx], name="mu_h%d" % (idx + 1), **l_hidden_kwargs, ) idx += 1 if bn: l_hidden = L.batch_norm(l_hidden) l_mu = L.DenseLayer( l_hidden, num_units=action_dim, nonlinearity=tf.nn.tanh, name="mu_h%d" % (idx + 1), **l_output_kwargs, ) L_var, V_var, mu_var = L.get_output([l_L, l_V, l_mu], deterministic=True) V_var = tf.reshape(V_var, (-1, )) # compute advantage L_mat_var = self.get_L_sym(L_var) P_var = self.get_P_sym(L_mat_var) A_var = self.get_A_sym(P_var, mu_var, l_action.input_var) # compute Q Q_var = A_var + V_var # compute expected Q under Gaussian policy e_A_var = self.get_e_A_sym(P_var, mu_var, l_policy_mu.input_var, l_policy_sigma.input_var) e_Q_var = e_A_var + V_var self._f_qval = tensor_utils.compile_function( [l_obs.input_var, l_action.input_var], Q_var) self._f_e_qval = tensor_utils.compile_function([ l_obs.input_var, l_policy_mu.input_var, l_policy_sigma.input_var ], e_Q_var) self._L_layer = l_L self._V_layer = l_V self._mu_layer = l_mu self._obs_layer = l_obs self._action_layer = l_action self._policy_mu_layer = l_policy_mu self._policy_sigma_layer = l_policy_sigma self._output_nonlinearity = output_nonlinearity self.init_policy() LayersPowered.__init__(self, [l_L, l_V, l_mu])
def __init__( self, name, output_dim, hidden_sizes, hidden_nonlinearity, dropout_prob, output_nonlinearity, hidden_W_init=L.XavierUniformInitializer(), hidden_b_init=tf.zeros_initializer(), output_W_init=L.XavierUniformInitializer(), output_b_init=tf.zeros_initializer(), input_var=None, input_layer=None, input_shape=None, batch_normalization=False, weight_normalization=False, ): Serializable.quick_init(self, locals()) with tf.variable_scope(name): if input_layer is None: l_in = L.InputLayer(shape=(None, ) + input_shape, input_var=input_var, name="input") else: l_in = input_layer self._layers = [l_in] ##applying dropout on all layers? l_hid_dropout_input = L.DropoutLayer(l_in, p=dropout_prob) l_hid = l_hid_dropout_input # l_hid = l_in if batch_normalization: l_hid = L.batch_norm(l_hid) for idx, hidden_size in enumerate(hidden_sizes): l_hid = L.DenseLayer(l_hid, num_units=hidden_size, nonlinearity=hidden_nonlinearity, name="hidden_%d" % idx, W=hidden_W_init, b=hidden_b_init, weight_normalization=weight_normalization) if batch_normalization: l_hid = L.batch_norm(l_hid) self._layers.append(l_hid) ###applying dropout to the last hidden layer? l_hid_dropout = L.DropoutLayer(l_hid, p=dropout_prob) l_out = L.DenseLayer(l_hid_dropout, num_units=output_dim, nonlinearity=output_nonlinearity, name="output", W=output_W_init, b=output_b_init, weight_normalization=weight_normalization) # l_out = L.DenseLayer( # l_hid, # num_units=output_dim, # nonlinearity=output_nonlinearity, # name="output", # W=output_W_init, # b=output_b_init, # weight_normalization=weight_normalization # ) #Alternative, making output layer the dropout layer # l_out = L.DropoutLayer(l_hid, p=dropout_prob) if batch_normalization: l_out = L.batch_norm(l_out) self._layers.append(l_out) self._l_in = l_in self._l_out = l_out # self._input_var = l_in.input_var self._output = L.get_output(l_out) LayersPowered.__init__(self, l_out)
def __init__( self, env_spec, name='qnet', hidden_sizes=(32, 32), hidden_nonlinearity=tf.nn.relu, action_merge_layer=-2, output_nonlinearity=None, hidden_W_init=L.XavierUniformInitializer(), hidden_b_init=L.ZerosInitializer(), output_W_init=L.XavierUniformInitializer(), output_b_init=L.ZerosInitializer(), c=1.0, # temperature variable for stochastic policy bn=False): Serializable.quick_init(self, locals()) assert env_spec.action_space.is_discrete self._n = env_spec.action_space.n self._c = c self._env_spec = env_spec with tf.variable_scope(name): l_obs = L.InputLayer(shape=(None, env_spec.observation_space.flat_dim), name="obs") l_action = L.InputLayer(shape=(None, env_spec.action_space.n), var_type=tf.uint8, name="actions") n_layers = len(hidden_sizes) + 1 l_hidden = l_obs for idx, size in enumerate(hidden_sizes): if bn: l_hidden = L.batch_norm(l_hidden) l_hidden = L.DenseLayer(l_hidden, num_units=size, W=hidden_W_init, b=hidden_b_init, nonlinearity=hidden_nonlinearity, name="h%d" % (idx + 1)) l_output_vec = L.DenseLayer(l_hidden, num_units=env_spec.action_space.n, W=output_W_init, b=output_b_init, nonlinearity=output_nonlinearity, name="output") output_vec_var = L.get_output(l_output_vec, deterministic=True) output_var = tf.reduce_sum( output_vec_var * tf.to_float(l_action.input_var), 1) self._f_qval = tensor_utils.compile_function( [l_obs.input_var, l_action.input_var], output_var) self._f_qval_vec = tensor_utils.compile_function([l_obs.input_var], output_vec_var) self._output_vec_layer = l_output_vec self._obs_layer = l_obs self._action_layer = l_action self._output_nonlinearity = output_nonlinearity self.init_policy() LayersPowered.__init__(self, [l_output_vec])