def __init__(self, name, input_shape, output_dim, hidden_dim, hidden_nonlinearity=tf.nn.relu, lstm_layer_cls=L.LSTMLayer, output_nonlinearity=None, input_var=None, input_layer=None, forget_bias=1.0, use_peepholes=False, layer_args=None): with tf.variable_scope(name): if input_layer is None: l_in = L.InputLayer(shape=(None, None) + input_shape, input_var=input_var, name="input") else: l_in = input_layer l_step_input = L.InputLayer(shape=(None,) + input_shape, name="step_input") # contains previous hidden and cell state l_step_prev_state = L.InputLayer(shape=(None, hidden_dim * 2), name="step_prev_state") if layer_args is None: layer_args = dict() l_lstm = lstm_layer_cls(l_in, num_units=hidden_dim, hidden_nonlinearity=hidden_nonlinearity, hidden_init_trainable=False, name="lstm", forget_bias=forget_bias, cell_init_trainable=False, use_peepholes=use_peepholes, **layer_args) l_lstm_flat = L.ReshapeLayer( l_lstm, shape=(-1, hidden_dim), name="lstm_flat" ) l_output_flat = L.DenseLayer( l_lstm_flat, num_units=output_dim, nonlinearity=output_nonlinearity, name="output_flat" ) l_output = L.OpLayer( l_output_flat, op=lambda flat_output, l_input: tf.reshape(flat_output, tf.stack((tf.shape(l_input)[0], tf.shape(l_input)[1], -1))), shape_op=lambda flat_output_shape, l_input_shape: (l_input_shape[0], l_input_shape[1], flat_output_shape[-1]), extras=[l_in], name="output" ) l_step_state = l_lstm.get_step_layer(l_step_input, l_step_prev_state, name="step_state") l_step_hidden = L.SliceLayer(l_step_state, indices=slice(hidden_dim), name="step_hidden") l_step_cell = L.SliceLayer(l_step_state, indices=slice(hidden_dim, None), name="step_cell") l_step_output = L.DenseLayer( l_step_hidden, num_units=output_dim, nonlinearity=output_nonlinearity, W=l_output_flat.W, b=l_output_flat.b, name="step_output" ) self._l_in = l_in self._hid_init_param = l_lstm.h0 self._cell_init_param = l_lstm.c0 self._l_lstm = l_lstm self._l_out = l_output self._l_step_input = l_step_input self._l_step_prev_state = l_step_prev_state self._l_step_hidden = l_step_hidden self._l_step_cell = l_step_cell self._l_step_state = l_step_state self._l_step_output = l_step_output self._hidden_dim = hidden_dim
def __init__(self, env_spec, hidden_sizes=(32, 32), hidden_nonlinearity=tf.nn.relu, action_merge_layer=-2, output_nonlinearity=None, bn=False, dropout=.05): Serializable.quick_init(self, locals()) l_obs = L.InputLayer(shape=(None, env_spec.observation_space.flat_dim), name="obs") l_action = L.InputLayer(shape=(None, env_spec.action_space.flat_dim), name="actions") n_layers = len(hidden_sizes) + 1 if n_layers > 1: action_merge_layer = \ (action_merge_layer % n_layers + n_layers) % n_layers else: action_merge_layer = 1 l_hidden = l_obs for idx, size in enumerate(hidden_sizes): if bn: l_hidden = batch_norm(l_hidden) if idx == action_merge_layer: l_hidden = L.ConcatLayer([l_hidden, l_action]) l_hidden = L.DenseLayer(l_hidden, num_units=size, nonlinearity=hidden_nonlinearity, name="h%d" % (idx + 1)) l_hidden = L.DropoutLayer(l_hidden, dropout) if action_merge_layer == n_layers: l_hidden = L.ConcatLayer([l_hidden, l_action]) l_output = L.DenseLayer(l_hidden, num_units=1, nonlinearity=output_nonlinearity, name="output") output_var = L.get_output(l_output, deterministic=True) output_var_drop = L.get_output(l_output, deterministic=False) self._f_qval = tensor_utils.compile_function( [l_obs.input_var, l_action.input_var], output_var) self._f_qval_drop = tensor_utils.compile_function( [l_obs.input_var, l_action.input_var], output_var_drop) self._output_layer = l_output self._obs_layer = l_obs self._action_layer = l_action self._output_nonlinearity = output_nonlinearity LayersPowered.__init__(self, [l_output])
def __init__( self, name, output_dim, hidden_sizes, hidden_nonlinearity, output_nonlinearity, hidden_W_init=L.XavierUniformInitializer(), hidden_b_init=tf.zeros_initializer(), output_W_init=L.XavierUniformInitializer(), output_b_init=tf.zeros_initializer(), input_var=None, input_layer=None, input_shape=None, batch_normalization=False, weight_normalization=False, ): Serializable.quick_init(self, locals()) with tf.variable_scope(name): if input_layer is None: l_in = L.InputLayer(shape=(None, ) + input_shape, input_var=input_var, name="input") else: l_in = input_layer self._layers = [l_in] l_hid = l_in if batch_normalization: l_hid = L.batch_norm(l_hid) for idx, hidden_size in enumerate(hidden_sizes): l_hid = L.DenseLayer(l_hid, num_units=hidden_size, nonlinearity=hidden_nonlinearity, name="hidden_%d" % idx, W=hidden_W_init, b=hidden_b_init, weight_normalization=weight_normalization) if batch_normalization: l_hid = L.batch_norm(l_hid) self._layers.append(l_hid) l_out = L.DenseLayer(l_hid, num_units=output_dim, nonlinearity=output_nonlinearity, name="output", W=output_W_init, b=output_b_init, weight_normalization=weight_normalization) if batch_normalization: l_out = L.batch_norm(l_out) self._layers.append(l_out) self._l_in = l_in self._l_out = l_out # self._input_var = l_in.input_var self._output = L.get_output(l_out) LayersPowered.__init__(self, l_out)
def __init__(self, name, input_shape, output_dim, hidden_dim, hidden_nonlinearity=tf.nn.relu, gru_layer_cls=L.GRULayer, output_nonlinearity=None, input_var=None, input_layer=None, layer_args=None): with tf.variable_scope(name): if input_layer is None: l_in = L.InputLayer(shape=(None, None) + input_shape, input_var=input_var, name="input") else: l_in = input_layer l_step_input = L.InputLayer(shape=(None,) + input_shape, name="step_input") l_step_prev_state = L.InputLayer(shape=(None, hidden_dim), name="step_prev_state") if layer_args is None: layer_args = dict() l_gru = gru_layer_cls(l_in, num_units=hidden_dim, hidden_nonlinearity=hidden_nonlinearity, hidden_init_trainable=False, name="gru", **layer_args) l_gru_flat = L.ReshapeLayer( l_gru, shape=(-1, hidden_dim), name="gru_flat" ) l_output_flat = L.DenseLayer( l_gru_flat, num_units=output_dim, nonlinearity=output_nonlinearity, name="output_flat" ) l_output = L.OpLayer( l_output_flat, op=lambda flat_output, l_input: tf.reshape(flat_output, tf.stack((tf.shape(l_input)[0], tf.shape(l_input)[1], -1))), shape_op=lambda flat_output_shape, l_input_shape: (l_input_shape[0], l_input_shape[1], flat_output_shape[-1]), extras=[l_in], name="output" ) l_step_state = l_gru.get_step_layer(l_step_input, l_step_prev_state, name="step_state") l_step_hidden = l_step_state l_step_output = L.DenseLayer( l_step_hidden, num_units=output_dim, nonlinearity=output_nonlinearity, W=l_output_flat.W, b=l_output_flat.b, name="step_output" ) self._l_in = l_in self._hid_init_param = l_gru.h0 self._l_gru = l_gru self._l_out = l_output self._l_step_input = l_step_input self._l_step_prev_state = l_step_prev_state self._l_step_hidden = l_step_hidden self._l_step_state = l_step_state self._l_step_output = l_step_output self._hidden_dim = hidden_dim
def create_MLP( self, name, output_dim, hidden_sizes, hidden_nonlinearity, output_nonlinearity, hidden_W_init=L.XavierUniformInitializer(), hidden_b_init=tf.zeros_initializer, output_W_init=L.XavierUniformInitializer(), output_b_init=tf.zeros_initializer, input_var=None, input_layer=None, input_shape=None, batch_normalization=False, weight_normalization=False, ): with tf.variable_scope(name): if input_layer is None: l_in = L.InputLayer(shape=(None, ) + input_shape, input_var=input_var, name="input") else: l_in = input_layer all_layers = [l_in] l_hid = l_in if batch_normalization: l_hid = L.batch_norm(l_hid) for idx, hidden_size in enumerate(hidden_sizes): l_hid = L.DenseLayer(l_hid, num_units=hidden_size, nonlinearity=hidden_nonlinearity, name="hidden_%d" % idx, W=hidden_W_init, b=hidden_b_init, weight_normalization=weight_normalization) if batch_normalization: l_hid = L.batch_norm(l_hid) all_layers.append(l_hid) l_out = L.DenseLayer(l_hid, num_units=output_dim, nonlinearity=output_nonlinearity, name="output", W=output_W_init, b=output_b_init, weight_normalization=weight_normalization) if batch_normalization: l_out = L.batch_norm(l_out) all_layers.append(l_out) output = L.get_output(l_out) # returns layers(), input_layer, output_layer, input_var, output return all_layers, l_in, l_out, l_in.input_var, output
def __init__(self, env_spec, name='QuadraticPhinet', hidden_sizes=(32, 32), hidden_nonlinearity=tf.nn.relu, output_nonlinearity=None, vs_form=None, bn=False, A=None, init_a=1.0, a_parameterization='exp'): Serializable.quick_init(self, locals()) assert not env_spec.action_space.is_discrete self._env_spec = env_spec self.vs_form = vs_form with tf.variable_scope(name): obs_dim = env_spec.observation_space.flat_dim action_dim = env_spec.action_space.flat_dim l_act = L.InputLayer(shape=(None, action_dim), name="action") action_var = l_act.input_var l_obs = L.InputLayer(shape=(None, obs_dim), name="obs") self.obs_rms = RunningMeanStd(shape=(obs_dim, )) obz = L.NormalizeLayer(l_obs, rms=self.obs_rms) l_hidden = l_obs hidden_sizes += (action_dim, ) for idx, size in enumerate(hidden_sizes): if bn: l_hidden = batch_norm(l_hidden) l_hidden = L.DenseLayer(l_hidden, num_units=size, nonlinearity=hidden_nonlinearity, name="h%d" % (idx + 1)) obs_var = l_obs.input_var fs = l_hidden # fs_network.output_layer if A is not None: l_A_param = A.output_layer else: if a_parameterization == 'exp': init_a_param = np.log(init_a) - .5 elif a_parameterization == 'softplus': init_a_param = np.log(np.exp(init_a) - 1) else: raise NotImplementedError l_log_A = L.ParamLayer( l_obs, num_units=action_dim, param=tf.constant_initializer(init_a_param), name="diagonal_a_matrix", trainable=True) if vs_form is not None: raise NotImplementedError self._l_log_A = l_log_A self.a_parameterization = a_parameterization self.fs = fs if vs_form is not None: self._output_vs = vs LayersPowered.__init__( self, [self.fs, self._l_log_A, self._output_vs]) else: LayersPowered.__init__(self, [self.fs, self._l_log_A]) output_var = self.get_phival_sym(obs_var, action_var) self._f_phival = tensor_utils.compile_function( inputs=[obs_var, action_var], outputs=output_var)
def make_network_image(self, dim_input, dim_output, nn_input=None, target=None): """ An example a network in tf that has both state and image inputs. Args: dim_input: Dimensionality of input. expecting 2d tuple (num_frames x num_batches) dim_output: Dimensionality of the output. batch_size: Batch size. network_config: dictionary of network structure parameters Returns: A tfMap object that stores inputs, outputs, and scalar loss. """ if dim_input[0] != 1: raise Exception( "Currently don't support concatenating timesteps for images") n_mlp_layers = 2 layer_size = 128 dim_hidden = (n_mlp_layers - 1) * [layer_size] dim_hidden.append(dim_output) pool_size = 2 filter_size = 3 num_filters = [5, 5] #TODO: don't do this grossness if nn_input is None: nn_input, _ = self.get_input_layer_image(dim_input[1], dim_output) if target is None: _, target = self.get_input_layer_image(dim_input[1], dim_output) conv_filters = [5, 5] conv_filter_sizes = [3, 3] conv_pads = ['SAME', 'SAME'] max_pool_sizes = [2, 2] conv_strides = [1, 1] hidden_sizes = [100, 100] hidden_nonlinearity = tf.nn.relu output_nonlinearity = None l_in = L.InputLayer(shape=tuple(nn_input.get_shape().as_list()), input_var=nn_input) l_hid = L.reshape(l_in, ([0], ) + dim_input[1], name="reshape_input") for idx, conv_filter, filter_size, stride, pad, max_pool_size in zip( range(len(conv_filters)), conv_filters, conv_filter_sizes, conv_strides, conv_pads, max_pool_sizes): l_hid = L.Conv2DLayer( l_hid, num_filters=conv_filter, filter_size=filter_size, stride=(stride, stride), pad=pad, nonlinearity=hidden_nonlinearity, name="conv_hidden_%d" % idx, ) if max_pool_size is not None: l_hid = L.Pool2DLayer(l_hid, max_pool_size, pad="SAME") l_hid = L.flatten(l_hid, name="conv_flatten") for idx, hidden_size in enumerate(hidden_sizes): l_hid = L.DenseLayer( l_hid, num_units=hidden_size, nonlinearity=hidden_nonlinearity, name="hidden_%d" % idx, ) fc_output = L.get_output( L.DenseLayer( l_hid, num_units=dim_output, nonlinearity=output_nonlinearity, name="output", )) loss, optimizer = self.get_loss_layer(pred=fc_output, target_output=target) self.class_target = target self.nn_input = nn_input self.discrimination_logits = fc_output self.optimizer = optimizer self.loss = loss label_accuracy = tf.equal( tf.round(tf.nn.sigmoid(self.discrimination_logits)), tf.round(self.class_target)) self.label_accuracy = tf.reduce_mean( tf.cast(label_accuracy, tf.float32)) self.mse = tf.reduce_mean( tf.nn.l2_loss( tf.nn.sigmoid(self.discrimination_logits) - self.class_target)) ones = tf.ones_like(self.class_target) true_positives = tf.round(tf.nn.sigmoid( self.discrimination_logits)) * tf.round(self.class_target) predicted_positives = tf.round( tf.nn.sigmoid(self.discrimination_logits)) false_negatives = tf.logical_not( tf.logical_xor( tf.equal(tf.round(tf.nn.sigmoid(self.discrimination_logits)), ones), tf.equal(tf.round(self.class_target), ones))) self.label_precision = tf.reduce_sum( tf.cast(true_positives, tf.float32)) / tf.reduce_sum( tf.cast(predicted_positives, tf.float32)) self.label_recall = tf.reduce_sum(tf.cast( true_positives, tf.float32)) / ( tf.reduce_sum(tf.cast(true_positives, tf.float32)) + tf.reduce_sum(tf.cast(false_negatives, tf.float32)))
def __init__(self, name, input_shape, output_dim, conv_filters, conv_filter_sizes, conv_strides, conv_pads, hidden_sizes, hidden_nonlinearity, output_nonlinearity, hidden_W_init=L.XavierUniformInitializer(), hidden_b_init=tf.zeros_initializer(), output_W_init=L.XavierUniformInitializer(), output_b_init=tf.zeros_initializer(), input_var=None, input_layer=None, batch_normalization=False, weight_normalization=False): Serializable.quick_init(self, locals()) """ A network composed of several convolution layers followed by some fc layers. input_shape: (width,height,channel) HOWEVER, network inputs are assumed flattened. This network will first unflatten the inputs and then apply the standard convolutions and so on. conv_filters: a list of numbers of convolution kernel conv_filter_sizes: a list of sizes (int) of the convolution kernels conv_strides: a list of strides (int) of the conv kernels conv_pads: a list of pad formats (either 'SAME' or 'VALID') hidden_nonlinearity: a nonlinearity from tf.nn, shared by all conv and fc layers hidden_sizes: a list of numbers of hidden units for all fc layers """ with tf.variable_scope(name): if input_layer is not None: l_in = input_layer l_hid = l_in elif len(input_shape) == 3: l_in = L.InputLayer(shape=(None, np.prod(input_shape)), input_var=input_var, name="input") l_hid = L.reshape(l_in, ([0],) + input_shape, name="reshape_input") elif len(input_shape) == 2: l_in = L.InputLayer(shape=(None, np.prod(input_shape)), input_var=input_var, name="input") input_shape = (1,) + input_shape l_hid = L.reshape(l_in, ([0],) + input_shape, name="reshape_input") else: l_in = L.InputLayer(shape=(None,) + input_shape, input_var=input_var, name="input") l_hid = l_in if batch_normalization: l_hid = L.batch_norm(l_hid) critical_size = hidden_sizes[0] for idx, conv_filter, filter_size, stride, pad in zip(range(len(conv_filters)), conv_filters, conv_filter_sizes, conv_strides, conv_pads, ): l_hid = L.Conv2DLayer( l_hid, num_filters=conv_filter, filter_size=filter_size, stride=(stride, stride), pad=pad, nonlinearity=hidden_nonlinearity, name="SL_conv_hidden_%d" % idx, weight_normalization=weight_normalization, ) if batch_normalization: l_hid = L.batch_norm(l_hid) l_hid = L.flatten(l_hid, name="conv_flatten") critical_layer = L.DenseLayer( l_hid, num_units=hidden_sizes[0], nonlinearity=None, name="SL_fc", W=hidden_W_init, b=hidden_b_init, weight_normalization=weight_normalization, ) #critical_layer = L.flatten(critical_layer) # if output_nonlinearity == L.spatial_expected_softmax: # assert len(hidden_sizes) == 0 # assert output_dim == conv_filters[-1] * 2 # l_hid.nonlinearity = tf.identity # l_out = L.SpatialExpectedSoftmaxLayer(l_hid) self.actValues = L.get_output(critical_layer) #list_rem = hidden_sizes[1:] #####Forward pass block################################# with tf.variable_scope("PG"): # fcFor = L.DenseLayer( # critical_layer, # num_units = hidden_sizes[1], # nonlinearity=hidden_nonlinearity, # name="pgLayer_init", # W=hidden_W_init, # b=hidden_b_init, # weight_normalization=weight_normalization, # ) fc_1 = L.DenseLayer( critical_layer, num_units=hidden_sizes[1], nonlinearity=hidden_nonlinearity, name="pgLayer_1", W=hidden_W_init, b=hidden_b_init, weight_normalization=weight_normalization, ) fc_2 = L.DenseLayer( fc_1, num_units=hidden_sizes[2], nonlinearity=hidden_nonlinearity, name="pgLayer_2" , W=hidden_W_init, b=hidden_b_init, weight_normalization=weight_normalization, ) if batch_normalization: fc_2 = L.batch_norm(fcFor) fcOut = L.DenseLayer( fc_2, num_units=output_dim, nonlinearity=output_nonlinearity, name="output", W=output_W_init, b=output_b_init, weight_normalization=weight_normalization, ) if batch_normalization: fcOut = L.batch_norm(fcOut) ################################################### self.actVariable = tf.Variable(initial_value = tf.zeros([ 10000, 32], dtype = tf.float32),name = "act_var1", trainable = True) bcOut = fcOut.get_output_for(fc_2.get_output_for(fc_1.get_output_for(self.actVariable))) self.bcOut = bcOut backOutLayer = L.InputLayer(shape = (), input_var= bcOut , name="OutputLayer") # shape is (actVariable[0] , 2) self._l_in = l_in self.forwardOutLayer = fcOut self.backOutLayer = backOutLayer outLayers = [fcOut, backOutLayer] # self._input_var = l_in.input_var LayersPowered.__init__(self, outLayers)
def __init__( self, env_spec, name='qnet', hidden_sizes=(32, 32), hidden_nonlinearity=tf.nn.relu, action_merge_layer=-2, output_nonlinearity=None, hidden_W_init=L.XavierUniformInitializer(), hidden_b_init=tf.zeros_initializer, output_W_init=L.XavierUniformInitializer(), output_b_init=tf.zeros_initializer, bn=False): Serializable.quick_init(self, locals()) with tf.variable_scope(name): l_obs = L.InputLayer(shape=(None, env_spec.observation_space.flat_dim), name="obs") l_action = L.InputLayer(shape=(None, env_spec.action_space.flat_dim), name="actions") n_layers = len(hidden_sizes) + 1 if n_layers > 1: action_merge_layer = \ (action_merge_layer % n_layers + n_layers) % n_layers else: action_merge_layer = 1 l_hidden = l_obs for idx, size in enumerate(hidden_sizes): if bn: l_hidden = L.batch_norm(l_hidden) if idx == action_merge_layer: l_hidden = L.ConcatLayer([l_hidden, l_action]) l_hidden = L.DenseLayer( l_hidden, num_units=size, W=hidden_W_init, b=hidden_b_init, nonlinearity=hidden_nonlinearity, name="h%d" % (idx + 1) ) if action_merge_layer == n_layers: l_hidden = L.ConcatLayer([l_hidden, l_action]) l_output = L.DenseLayer( l_hidden, num_units=1, W=output_W_init, b=output_b_init, nonlinearity=output_nonlinearity, name="output" ) #output_var = L.get_output(l_output, deterministic=True).flatten() output_var = tf.reshape(L.get_output(l_output, deterministic=True),(-1,)) self._f_qval = tensor_utils.compile_function([l_obs.input_var, l_action.input_var], output_var) self._output_layer = l_output self._obs_layer = l_obs self._action_layer = l_action self._output_nonlinearity = output_nonlinearity LayersPowered.__init__(self, [l_output])
def __init__(self, env_spec, name='nafqnet', hidden_sizes=(32, 32), hidden_nonlinearity=tf.nn.relu, action_merge_layer=0, output_nonlinearity=None, hidden_W_init=L.XavierUniformInitializer(), hidden_b_init=L.ZerosInitializer(), output_W_init=L.XavierUniformInitializer(), output_b_init=L.ZerosInitializer(), bn=False): Serializable.quick_init(self, locals()) assert not env_spec.action_space.is_discrete action_dim = env_spec.action_space.flat_dim self._action_dim = action_dim self._env_spec = env_spec n_layers = len(hidden_sizes) action_merge_layer = \ (action_merge_layer % n_layers + n_layers) % n_layers with tf.variable_scope(name): l_obs = L.InputLayer(shape=(None, env_spec.observation_space.flat_dim), name="obs") l_action = L.InputLayer(shape=(None, env_spec.action_space.flat_dim), name="actions") l_policy_mu = L.InputLayer(shape=(None, action_dim), name="policy_mu") l_policy_sigma = L.InputLayer(shape=(None, action_dim, action_dim), name="policy_sigma") l_hidden = l_obs idx = 0 l_hidden_kwargs = dict( W=hidden_W_init, b=hidden_b_init, nonlinearity=hidden_nonlinearity, ) l_output_kwargs = dict( W=output_W_init, b=output_b_init, ) while idx < action_merge_layer: if bn: l_hidden = L.batch_norm(l_hidden) l_hidden = L.DenseLayer( l_hidden, num_units=hidden_sizes[idx], name="h%d" % (idx + 1), **l_hidden_kwargs, ) idx += 1 _idx = idx _l_hidden = l_hidden # compute L network while idx < n_layers: if bn: l_hidden = L.batch_norm(l_hidden) l_hidden = L.DenseLayer( l_hidden, num_units=hidden_sizes[idx], name="L_h%d" % (idx + 1), **l_hidden_kwargs, ) idx += 1 l_L = L.DenseLayer( l_hidden, num_units=action_dim**2, nonlinearity=None, name="L_h%d" % (idx + 1), **l_output_kwargs, ) # compute V network idx = _idx l_hidden = _l_hidden while idx < n_layers: if bn: l_hidden = L.batch_norm(l_hidden) l_hidden = L.DenseLayer( l_hidden, num_units=hidden_sizes[idx], name="V_h%d" % (idx + 1), **l_hidden_kwargs, ) idx += 1 l_V = L.DenseLayer( l_hidden, num_units=1, nonlinearity=None, name="V_h%d" % (idx + 1), **l_output_kwargs, ) # compute mu network idx = _idx l_hidden = _l_hidden while idx < n_layers: if bn: l_hidden = L.batch_norm(l_hidden) l_hidden = L.DenseLayer( l_hidden, num_units=hidden_sizes[idx], name="mu_h%d" % (idx + 1), **l_hidden_kwargs, ) idx += 1 if bn: l_hidden = L.batch_norm(l_hidden) l_mu = L.DenseLayer( l_hidden, num_units=action_dim, nonlinearity=tf.nn.tanh, name="mu_h%d" % (idx + 1), **l_output_kwargs, ) L_var, V_var, mu_var = L.get_output([l_L, l_V, l_mu], deterministic=True) V_var = tf.reshape(V_var, (-1, )) # compute advantage L_mat_var = self.get_L_sym(L_var) P_var = self.get_P_sym(L_mat_var) A_var = self.get_A_sym(P_var, mu_var, l_action.input_var) # compute Q Q_var = A_var + V_var # compute expected Q under Gaussian policy e_A_var = self.get_e_A_sym(P_var, mu_var, l_policy_mu.input_var, l_policy_sigma.input_var) e_Q_var = e_A_var + V_var self._f_qval = tensor_utils.compile_function( [l_obs.input_var, l_action.input_var], Q_var) self._f_e_qval = tensor_utils.compile_function([ l_obs.input_var, l_policy_mu.input_var, l_policy_sigma.input_var ], e_Q_var) self._L_layer = l_L self._V_layer = l_V self._mu_layer = l_mu self._obs_layer = l_obs self._action_layer = l_action self._policy_mu_layer = l_policy_mu self._policy_sigma_layer = l_policy_sigma self._output_nonlinearity = output_nonlinearity self.init_policy() LayersPowered.__init__(self, [l_L, l_V, l_mu])
def __init__(self, name, input_shape, output_dim, conv_filters, conv_filter_sizes, conv_strides, conv_pads, hidden_sizes, hidden_nonlinearity, output_nonlinearity, hidden_W_init=L.XavierUniformInitializer(), hidden_b_init=tf.zeros_initializer(), output_W_init=L.XavierUniformInitializer(), output_b_init=tf.zeros_initializer(), input_var=None, input_layer=None, batch_normalization=False, weight_normalization=False): Serializable.quick_init(self, locals()) """ A network composed of several convolution layers followed by some fc layers. input_shape: (width,height,channel) HOWEVER, network inputs are assumed flattened. This network will first unflatten the inputs and then apply the standard convolutions and so on. conv_filters: a list of numbers of convolution kernel conv_filter_sizes: a list of sizes (int) of the convolution kernels conv_strides: a list of strides (int) of the conv kernels conv_pads: a list of pad formats (either 'SAME' or 'VALID') hidden_nonlinearity: a nonlinearity from tf.nn, shared by all conv and fc layers hidden_sizes: a list of numbers of hidden units for all fc layers """ with tf.variable_scope(name): if input_layer is not None: l_in = input_layer l_hid = l_in elif len(input_shape) == 3: l_in = L.InputLayer(shape=(None, np.prod(input_shape)), input_var=input_var, name="input") l_hid = L.reshape(l_in, ([0], ) + input_shape, name="reshape_input") elif len(input_shape) == 2: l_in = L.InputLayer(shape=(None, np.prod(input_shape)), input_var=input_var, name="input") input_shape = (1, ) + input_shape l_hid = L.reshape(l_in, ([0], ) + input_shape, name="reshape_input") else: l_in = L.InputLayer(shape=(None, ) + input_shape, input_var=input_var, name="input") l_hid = l_in if batch_normalization: l_hid = L.batch_norm(l_hid) for idx, conv_filter, filter_size, stride, pad in zip( range(len(conv_filters)), conv_filters, conv_filter_sizes, conv_strides, conv_pads, ): l_hid = L.Conv2DLayer( l_hid, num_filters=conv_filter, filter_size=filter_size, stride=(stride, stride), pad=pad, nonlinearity=hidden_nonlinearity, name="conv_hidden_%d" % idx, weight_normalization=weight_normalization, ) if batch_normalization: l_hid = L.batch_norm(l_hid) if output_nonlinearity == L.spatial_expected_softmax: assert len(hidden_sizes) == 0 assert output_dim == conv_filters[-1] * 2 l_hid.nonlinearity = tf.identity l_out = L.SpatialExpectedSoftmaxLayer(l_hid) else: l_hid = L.flatten(l_hid, name="conv_flatten") for idx, hidden_size in enumerate(hidden_sizes): l_hid = L.DenseLayer( l_hid, num_units=hidden_size, nonlinearity=hidden_nonlinearity, name="hidden_%d" % idx, W=hidden_W_init, b=hidden_b_init, weight_normalization=weight_normalization, ) if batch_normalization: l_hid = L.batch_norm(l_hid) l_out = L.DenseLayer( l_hid, num_units=output_dim, nonlinearity=output_nonlinearity, name="output", W=output_W_init, b=output_b_init, weight_normalization=weight_normalization, ) if batch_normalization: l_out = L.batch_norm(l_out) self._l_in = l_in self._l_out = l_out # self._input_var = l_in.input_var LayersPowered.__init__(self, l_out)
def __init__(self, name, output_dim, hidden_sizes, hidden_nonlinearity, output_nonlinearity, hidden_W_init=L.XavierUniformInitializer(), hidden_b_init=tf.zeros_initializer(), output_W_init=L.XavierUniformInitializer(), output_b_init=tf.zeros_initializer(), input_var=None, input_layer=None, input_shape=None, batch_normalization=False, weight_normalization=False, latent_dim=0, latent_shape=None, obs_shape=None): Serializable.quick_init(self, locals()) with tf.variable_scope(name): if input_layer is None: l_in = L.InputLayer(shape=(None, ) + input_shape, input_var=input_var, name="input") else: l_in = input_layer # latent_in = L.InputLayer(shape=(None,) + latent_shape, input_var=l_in.input_var[:, -latent_dim:], name='latent') # obs_in = L.InputLayer(shape=(None,) + obs_shape, input_var=l_in.input_var[:, :-latent_dim], name='obs_input') latent_in = L.SliceLayer(l_in, slice(-latent_dim, None, None), axis=-1) obs_in = L.SliceLayer(l_in, slice(0, -latent_dim, None), axis=-1) self._layers = [obs_in] l_hid = obs_in if batch_normalization: l_hid = L.batch_norm(l_hid) for idx, hidden_size in enumerate(hidden_sizes): l_hid = L.DenseLayer(l_hid, num_units=hidden_size, nonlinearity=hidden_nonlinearity, name="hidden_%d" % idx, W=hidden_W_init, b=hidden_b_init, weight_normalization=weight_normalization) if batch_normalization: l_hid = L.batch_norm(l_hid) self._layers.append(l_hid) l_latent_out = L.DenseLayer( latent_in, num_units=hidden_size, nonlinearity=hidden_nonlinearity, name="hidden_latent_0", W=hidden_W_init, b=hidden_b_init, weight_normalization=weight_normalization) if batch_normalization: l_latent_out = L.batch_norm(l_latent_out) self._layers.append(l_latent_out) l_hid = L.ElemwiseSumLayer([l_hid, l_latent_out]) # l_hid = L.OpLayer( # l_hid, # op=lambda l_hid, l_latent: # l_hid + l_latent, # shape_op=lambda l_hid_shape, l_latent_shape: # l_hid_shape, # extras=[l_latent_out], # name='sum_obs_latent') l_out = L.DenseLayer(l_hid, num_units=output_dim, nonlinearity=output_nonlinearity, name="output", W=output_W_init, b=output_b_init, weight_normalization=weight_normalization) if batch_normalization: l_out = L.batch_norm(l_out) self._layers.append(l_out) self._l_in = l_in self._l_out = l_out # self._input_var = l_in.input_var self._output = L.get_output(l_out) LayersPowered.__init__(self, l_out)
def __init__( self, name, output_dim, hidden_sizes, hidden_nonlinearity, hidden_W_init=L.XavierUniformInitializer(), hidden_b_init=tf.zeros_initializer(), output_W_init=L.XavierUniformInitializer(), output_b_init=tf.zeros_initializer(), input_var=None, input_layer=None, input_shape=None, batch_normalization=False, weight_normalization=False, ): Serializable.quick_init(self, locals()) with tf.variable_scope(name): if input_layer is None: assert input_shape is not None, \ "input_layer or input_shape must be supplied" l_in = L.InputLayer(shape=(None, ) + input_shape, input_var=input_var, name="input") else: l_in = input_layer self._layers = [l_in] l_hid = l_in if batch_normalization: l_hid = L.batch_norm(l_hid) for idx, hidden_size in enumerate(hidden_sizes): l_hid = L.DenseLayer(l_hid, num_units=hidden_size, nonlinearity=hidden_nonlinearity, name="hidden_%d" % idx, W=hidden_W_init, b=hidden_b_init, weight_normalization=weight_normalization) if batch_normalization: l_hid = L.batch_norm(l_hid) self._layers.append(l_hid) l_out_raw = L.DenseLayer(l_hid, num_units=output_dim, name="output", W=output_W_init, b=output_b_init, weight_normalization=weight_normalization) if batch_normalization: l_out_raw = L.batch_norm(l_out_raw) self._layers.append(l_out_raw) # mask assumed to occupy first output_dim elements def mask_op(X): return X[..., :output_dim] def mask_shape_op(old_shape): return old_shape[:-1] + (output_dim, ) mask = L.OpLayer(l_in, mask_op, shape_op=mask_shape_op) self._layers.append(mask) l_out = L.OpLayer(l_out_raw, masked_softmax, extras=[mask]) self._layers.append(l_out) self._l_in = l_in self._l_out = l_out # self._input_var = l_in.input_var self._output = L.get_output(l_out) LayersPowered.__init__(self, l_out)
def __init__(self, env_spec, name='Phinet', hidden_sizes=(32, 32), hidden_nonlinearity=tf.nn.relu, action_merge_layer=-2, output_nonlinearity=None, bn=False): Serializable.quick_init(self, locals()) assert not env_spec.action_space.is_discrete self._env_spec = env_spec with tf.variable_scope(name): l_obs = L.InputLayer(shape=(None, env_spec.observation_space.flat_dim), name="obs") l_action = L.InputLayer(shape=(None, env_spec.action_space.flat_dim), name="action") n_layers = len(hidden_sizes) + 1 if n_layers > 1: action_merge_layer = \ (action_merge_layer % n_layers + n_layers) % n_layers else: action_merge_layer = 1 # self.obs_rms = RunningMeanStd(shape=(env_spec.observation_space.flat_dim, )) # obz = L.NormalizeLayer(l_obs, rms=self.obs_rms, clip_min=-5., clip_max=5.) obz = l_obs obs_hidden = L.DenseLayer(obz, num_units=hidden_sizes[0], nonlinearity=hidden_nonlinearity, name="obs_h%d" % (0)) act_hidden = L.DenseLayer(l_action, num_units=hidden_sizes[0], nonlinearity=hidden_nonlinearity, name="act_h%d" % (0)) merge_hidden = L.OpLayer(obs_hidden, op=lambda x, y: x + y, shape_op=lambda x, y: x, extras=[act_hidden]) l_hidden = merge_hidden for idx, size in enumerate(hidden_sizes[1:]): if bn: l_hidden = batch_norm(l_hidden) l_hidden = L.DenseLayer(l_hidden, num_units=size, nonlinearity=hidden_nonlinearity, name="h%d" % (idx + 1)) # for idx, size in enumerate(hidden_sizes): # if bn: # l_hidden = batch_norm(l_hidden) # if idx == action_merge_layer: # l_hidden = L.ConcatLayer([l_hidden, l_action]) # l_hidden = L.DenseLayer( # l_hidden, # num_units=size, # nonlinearity=hidden_nonlinearity, # name="h%d" % (idx + 1) # ) # if action_merge_layer == n_layers: # l_hidden = L.ConcatLayer([l_hidden, l_action]) l_output = L.DenseLayer(l_hidden, num_units=1, nonlinearity=output_nonlinearity, name="output") output_var = L.get_output(l_output, deterministic=True) output_var = tf.reshape(output_var, (-1, )) self._f_phival = tensor_utils.compile_function( [l_obs.input_var, l_action.input_var], output_var) self._output_layer = l_output self._obs_layer = l_obs self._action_layer = l_action self.output_nonlinearity = output_nonlinearity LayersPowered.__init__(self, [l_output])
def __init__(self, name, input_dim, output_dim, hidden_sizes, hidden_nonlinearity, output_nonlinearity, vocab_size, embedding_size, hidden_W_init=L.xavier_init, hidden_b_init=tf.zeros_initializer, output_W_init=L.xavier_init, output_b_init=tf.zeros_initializer, has_other_input=True, input_var=None, input_layer=None, **kwargs): Serializable.quick_init(self, locals()) with tf.variable_scope(name): if input_layer is None: input_layer = L.InputLayer(shape=(None, input_dim), input_var=input_var, name="input") l_in = input_layer if has_other_input: # Slice apart l_other_in = L.SliceLayer(l_in, "slice_other", slice(0, input_dim - vocab_size), axis=-1) l_emb_in = L.SliceLayer(l_in, "slice_emb", slice(input_dim - vocab_size, input_dim), axis=-1) # HACK: This is cheap with small embedding matrices but will not scale well.. # Find a better way to lookup from this representation + mean-pool l_embs = MeanPoolEmbeddingLayer(l_emb_in, "embeddings", embedding_size) l_hidden_input = L.ConcatLayer([l_other_in, l_embs], "merge") else: l_hidden_input = l_in hidden_layers = [l_hidden_input] for i, hidden_size in enumerate(hidden_sizes): l_hid = L.DenseLayer(hidden_layers[-1], num_units=hidden_size, nonlinearity=hidden_nonlinearity, name="hidden_%i" % i, W=hidden_W_init, b=hidden_b_init) hidden_layers.append(l_hid) l_out = L.DenseLayer(hidden_layers[-1], num_units=output_dim, nonlinearity=output_nonlinearity, name="output", W=output_W_init, b=output_b_init) self.input_layer = l_in self.input_var = l_in.input_var self.output_layer = l_out LayersPowered.__init__(self, l_out)
def __init__(self, name, input_shape, output_dim, hidden_dims, hidden_nonlinearity=tf.nn.relu, output_nonlinearity=None, input_var=None, input_layer=None): with tf.variable_scope(name): if input_layer is None: l_in = L.InputLayer(shape=(None, None) + input_shape, input_var=input_var, name="input") else: l_in = input_layer l_step_input = L.InputLayer(shape=(None, ) + input_shape, name="step_input") l_step_prev_hiddens = [ L.InputLayer(shape=(None, hidden_dim), name="step_prev_hidden%i" % i) for i, hidden_dim in enumerate(hidden_dims) ] # Build the unrolled GRU network, which operates laterally, then # vertically below = l_in l_grus = [] for i, hidden_dim in enumerate(hidden_dims): l_gru = L.GRULayer(below, num_units=hidden_dim, hidden_nonlinearity=hidden_nonlinearity, hidden_init_trainable=False, name="gru%i" % i) l_grus.append(l_gru) below = l_gru # Convert final hidden layer to flat representation l_gru_flat = L.ReshapeLayer(l_grus[-1], shape=(-1, hidden_dims[-1]), name="gru_flat") l_output_flat = L.DenseLayer(l_gru_flat, num_units=output_dim, nonlinearity=output_nonlinearity, name="output_flat") l_output = L.OpLayer( l_output_flat, op=lambda flat_output, l_input: tf.reshape( flat_output, tf.pack((tf.shape(l_input)[0], tf.shape(l_input)[1], -1))), shape_op=lambda flat_output_shape, l_input_shape: (l_input_shape[0], l_input_shape[1], flat_output_shape[-1]), extras=[l_in], name="output") # Build a single step of the GRU network, which operates vertically # and is replicated laterally below = l_step_input l_step_hiddens = [] for i, (l_gru, prev_hidden) in enumerate(zip(l_grus, l_step_prev_hiddens)): l_step_hidden = L.GRUStepLayer([below, prev_hidden], "step_hidden%i" % i, l_gru) l_step_hiddens.append(l_step_hidden) below = l_step_hidden l_step_output = L.DenseLayer(l_step_hiddens[-1], num_units=output_dim, nonlinearity=output_nonlinearity, W=l_output_flat.W, b=l_output_flat.b, name="step_output") self._l_in = l_in self._hid_inits = [l_gru.h0 for l_gru in l_grus] self._l_grus = l_grus self._l_out = l_output self._l_step_input = l_step_input self._l_step_prev_hiddens = l_step_prev_hiddens self._l_step_hiddens = l_step_hiddens self._l_step_output = l_step_output
def __init__( self, env_spec, name='qnet', hidden_sizes=(32, 32), hidden_nonlinearity=tf.nn.relu, action_merge_layer=-2, output_nonlinearity=None, # hidden_W_init=L.XavierUniformInitializer(), # hidden_b_init=L.ZerosInitializer(), # output_W_init=L.XavierUniformInitializer(), # output_b_init=L.ZerosInitializer(), c=1.0, # temperature variable for stochastic policy bn=False): Serializable.quick_init(self, locals()) # assert env_spec.action_space.is_discrete self._n = 2 self._c = c self._env_spec = env_spec with tf.variable_scope(name): l_obs = L.InputLayer(shape=(None, env_spec.observation_space.flat_dim), name="obs") l_action = L.InputLayer(shape=(None, 2), var_type=tf.uint8, name="actions") n_layers = len(hidden_sizes) + 1 l_hidden = l_obs for idx, size in enumerate(hidden_sizes): if bn: l_hidden = L.batch_norm(l_hidden) l_hidden = L.DenseLayer( l_hidden, num_units=size, # W=hidden_W_init, # b=hidden_b_init, nonlinearity=hidden_nonlinearity, name="h%d" % (idx + 1)) l_output_vec = L.DenseLayer( l_hidden, num_units=2, # W=output_W_init, # b=output_b_init, nonlinearity=output_nonlinearity, name="output") output_vec_var = L.get_output(l_output_vec, deterministic=True) output_var = tf.reduce_sum( output_vec_var * tf.to_float(l_action.input_var), 1) self._f_qval = tensor_utils.compile_function( [l_obs.input_var, l_action.input_var], output_var) self._f_qval_vec = tensor_utils.compile_function([l_obs.input_var], output_vec_var) self._output_vec_layer = l_output_vec self._obs_layer = l_obs self._action_layer = l_action self._output_nonlinearity = output_nonlinearity self.init_policy() LayersPowered.__init__(self, [l_output_vec])
def __init__(self, name, input_shape, extra_input_shape, output_dim, hidden_sizes, conv_filters, conv_filter_sizes, conv_strides, conv_pads, extra_hidden_sizes=None, hidden_W_init=L.XavierUniformInitializer(), hidden_b_init=tf.zeros_initializer(), output_W_init=L.XavierUniformInitializer(), output_b_init=tf.zeros_initializer(), hidden_nonlinearity=tf.nn.relu, output_nonlinearity=None, input_var=None, input_layer=None): Serializable.quick_init(self, locals()) if extra_hidden_sizes is None: extra_hidden_sizes = [] with tf.variable_scope(name): input_flat_dim = np.prod(input_shape) extra_input_flat_dim = np.prod(extra_input_shape) total_input_flat_dim = input_flat_dim + extra_input_flat_dim if input_layer is None: l_in = L.InputLayer(shape=(None, total_input_flat_dim), input_var=input_var, name="input") else: l_in = input_layer l_conv_in = L.reshape(L.SliceLayer(l_in, indices=slice(input_flat_dim), name="conv_slice"), ([0], ) + input_shape, name="conv_reshaped") l_extra_in = L.reshape(L.SliceLayer(l_in, indices=slice( input_flat_dim, None), name="extra_slice"), ([0], ) + extra_input_shape, name="extra_reshaped") l_conv_hid = l_conv_in for idx, conv_filter, filter_size, stride, pad in zip( range(len(conv_filters)), conv_filters, conv_filter_sizes, conv_strides, conv_pads, ): l_conv_hid = L.Conv2DLayer( l_conv_hid, num_filters=conv_filter, filter_size=filter_size, stride=(stride, stride), pad=pad, nonlinearity=hidden_nonlinearity, name="conv_hidden_%d" % idx, ) l_extra_hid = l_extra_in for idx, hidden_size in enumerate(extra_hidden_sizes): l_extra_hid = L.DenseLayer( l_extra_hid, num_units=hidden_size, nonlinearity=hidden_nonlinearity, name="extra_hidden_%d" % idx, W=hidden_W_init, b=hidden_b_init, ) l_joint_hid = L.concat( [L.flatten(l_conv_hid, name="conv_hidden_flat"), l_extra_hid], name="joint_hidden") for idx, hidden_size in enumerate(hidden_sizes): l_joint_hid = L.DenseLayer( l_joint_hid, num_units=hidden_size, nonlinearity=hidden_nonlinearity, name="joint_hidden_%d" % idx, W=hidden_W_init, b=hidden_b_init, ) l_out = L.DenseLayer( l_joint_hid, num_units=output_dim, nonlinearity=output_nonlinearity, name="output", W=output_W_init, b=output_b_init, ) self._l_in = l_in self._l_out = l_out LayersPowered.__init__(self, [l_out], input_layers=[l_in])
def __init__(self, env_spec, name='MLPPhinet', hidden_sizes=(100, 100), hidden_nonlinearity=tf.nn.relu, action_merge_layer=-2, output_nonlinearity=None, vs_form=None, bn=False): Serializable.quick_init(self, locals()) assert not env_spec.action_space.is_discrete self._env_spec = env_spec self.vs_form = vs_form with tf.variable_scope(name): obs_dim = env_spec.observation_space.flat_dim action_dim = env_spec.action_space.flat_dim l_obs = L.InputLayer(shape=(None, obs_dim), name="obs") l_action = L.InputLayer(shape=(None, action_dim), name="action") self.obs_rms = RunningMeanStd(shape=(obs_dim, )) obz = L.NormalizeLayer(l_obs, rms=self.obs_rms, clip_min=-5., clip_max=5.) obs_hidden = L.DenseLayer(obz, num_units=hidden_sizes[0], nonlinearity=hidden_nonlinearity, name="obs_h%d" % (0)) print("hidden sizes...", hidden_sizes[0], hidden_sizes[1:]) act_hidden = L.DenseLayer(l_action, num_units=hidden_sizes[0], nonlinearity=hidden_nonlinearity, name="act_h%d" % (0)) merge_hidden = L.OpLayer(obs_hidden, op=lambda x, y: x + y, shape_op=lambda x, y: y, extras=[act_hidden]) l_hidden = merge_hidden for idx, size in enumerate(hidden_sizes[1:]): if bn: l_hidden = batch_norm(l_hidden) l_hidden = L.DenseLayer(l_hidden, num_units=size, nonlinearity=hidden_nonlinearity, name="h%d" % (idx + 1)) l_output = L.DenseLayer(l_hidden, num_units=1, nonlinearity=output_nonlinearity, name="output") if vs_form is not None: if vs_form == 'linear': vs = L.DenseLayer(l_obs, num_units=1, nonlinearity=None, name='vs') elif vs_form == 'mlp': vs = L.DenseLayer(l_obs, num_units=64, nonlinearity=tf.nn.relu, name='hidden_vs') vs = L.DenseLayer(vs, num_units=1, nonlinearity=None, name='vs') else: raise NotImplementedError output_var = L.get_output(l_output, deterministic=True) + \ L.get_output(vs, deterministic=True) output_var = tf.reshape(output_var, (-1, )) else: output_var = L.get_output(l_output, deterministic=True) output_var = tf.reshape(output_var, (-1, )) self._f_phival = tensor_utils.compile_function( inputs=[l_obs.input_var, l_action.input_var], outputs=output_var) self._output_layer = l_output self._obs_layer = l_obs self._action_layer = l_action self.output_nonlinearity = output_nonlinearity if vs_form is not None: self._output_vs = vs LayersPowered.__init__(self, [l_output, self._output_vs]) else: LayersPowered.__init__(self, [l_output])
def __init__(self, env_spec, name='qnet', hidden_sizes=(32, 32), hidden_nonlinearity=tf.nn.relu, action_merge_layer=-2, output_nonlinearity=None, eqf_use_full_qf=False, eqf_sample_size=1, mqprop=False, bn=False): Serializable.quick_init(self, locals()) assert not env_spec.action_space.is_discrete self._env_spec = env_spec with tf.variable_scope(name): l_obs = L.InputLayer(shape=(None, env_spec.observation_space.flat_dim), name="obs") l_action = L.InputLayer(shape=(None, env_spec.action_space.flat_dim), name="actions") n_layers = len(hidden_sizes) + 1 if n_layers > 1: action_merge_layer = \ (action_merge_layer % n_layers + n_layers) % n_layers else: action_merge_layer = 1 l_hidden = l_obs for idx, size in enumerate(hidden_sizes): if bn: l_hidden = batch_norm(l_hidden) if idx == action_merge_layer: l_hidden = L.ConcatLayer([l_hidden, l_action]) l_hidden = L.DenseLayer(l_hidden, num_units=size, nonlinearity=hidden_nonlinearity, name="h%d" % (idx + 1)) if action_merge_layer == n_layers: l_hidden = L.ConcatLayer([l_hidden, l_action]) l_output = L.DenseLayer(l_hidden, num_units=1, nonlinearity=output_nonlinearity, name="output") output_var = L.get_output(l_output, deterministic=True) output_var = tf.reshape(output_var, (-1, )) self._f_qval = tensor_utils.compile_function( [l_obs.input_var, l_action.input_var], output_var) self._output_layer = l_output self._obs_layer = l_obs self._action_layer = l_action self._output_nonlinearity = output_nonlinearity self.eqf_use_full_qf = eqf_use_full_qf self.eqf_sample_size = eqf_sample_size self.mqprop = mqprop LayersPowered.__init__(self, [l_output])
def __init__( self, name, output_dim, hidden_sizes, hidden_nonlinearity, dropout_prob, output_nonlinearity, hidden_W_init=L.XavierUniformInitializer(), hidden_b_init=tf.zeros_initializer(), output_W_init=L.XavierUniformInitializer(), output_b_init=tf.zeros_initializer(), input_var=None, input_layer=None, input_shape=None, batch_normalization=False, weight_normalization=False, ): Serializable.quick_init(self, locals()) with tf.variable_scope(name): if input_layer is None: l_in = L.InputLayer(shape=(None, ) + input_shape, input_var=input_var, name="input") else: l_in = input_layer self._layers = [l_in] ##applying dropout on all layers? l_hid_dropout_input = L.DropoutLayer(l_in, p=dropout_prob) l_hid = l_hid_dropout_input # l_hid = l_in if batch_normalization: l_hid = L.batch_norm(l_hid) for idx, hidden_size in enumerate(hidden_sizes): l_hid = L.DenseLayer(l_hid, num_units=hidden_size, nonlinearity=hidden_nonlinearity, name="hidden_%d" % idx, W=hidden_W_init, b=hidden_b_init, weight_normalization=weight_normalization) if batch_normalization: l_hid = L.batch_norm(l_hid) self._layers.append(l_hid) ###applying dropout to the last hidden layer? l_hid_dropout = L.DropoutLayer(l_hid, p=dropout_prob) l_out = L.DenseLayer(l_hid_dropout, num_units=output_dim, nonlinearity=output_nonlinearity, name="output", W=output_W_init, b=output_b_init, weight_normalization=weight_normalization) # l_out = L.DenseLayer( # l_hid, # num_units=output_dim, # nonlinearity=output_nonlinearity, # name="output", # W=output_W_init, # b=output_b_init, # weight_normalization=weight_normalization # ) #Alternative, making output layer the dropout layer # l_out = L.DropoutLayer(l_hid, p=dropout_prob) if batch_normalization: l_out = L.batch_norm(l_out) self._layers.append(l_out) self._l_in = l_in self._l_out = l_out # self._input_var = l_in.input_var self._output = L.get_output(l_out) LayersPowered.__init__(self, l_out)