def __call__(self, state): with tf.variable_scope(self.name): # Net self.width = 1 self.offset = 0 self.k = 0.07 self.theta = tf.get_variable( "theta", dtype=get_default_tf_dtype(), shape=(1, 1), initializer=tf.initializers.constant( self.from_sigm_to_theta(self.init_theta)), ) theta = self.width * ( 1 / (1 + tf.exp(-self.k * self.theta))) + self.offset # For taking actions self._pi = tf.concat( [ tf.tile(theta, (tf.shape(state)[0], 1)), tf.tile(1 - theta, (tf.shape(state)[0], 1)), ], axis=1, ) self._log_pi = tf.log(self._pi + self.epsilon_small) return self._pi, self._log_pi
def __init__( self, policy, inputQ, outputQ, n_actions, obs_size, id, total_n_samples, n_params, ): # Invoke parent constructor BEFORE doing anything!! Process.__init__(self) self.dtype = get_default_tf_dtype() self.env = Torcs(port=id) self.state_tf = tf.placeholder(self.dtype, (None, self.env.observation_space_size), name="states") self.action_tf = tf.placeholder(self.dtype, (None, self.env.action_space_size), name="actions") self.policy_tf = None self.policy = policy self.inputQ = inputQ self.outputQ = outputQ self.n_actions = n_actions self.id = id self.total_n_samples = total_n_samples self.n_params = n_params
def __init__(self, name="policy", init_theta=np.random.rand()): """ Builds a policy network and returns a node for pi and a node for logpi """ # net params super(OneParameterPolicy, self).__init__(name) self.sess = None self.default_dtype = get_default_tf_dtype() self.epsilon_small = 1e-20 self.action_space = 2 self.init_theta = init_theta
def __init__(self, policy, env, n_traj, inputQ, outputQ, n_actions, obs_size): # Invoke parent constructor BEFORE doing anything!! Process.__init__(self) self.dtype = get_default_tf_dtype() self.state_tf = tf.placeholder(self.dtype, (None, obs_size), name="states") self.policy = policy self.n_traj = n_traj self.inputQ = inputQ self.outputQ = outputQ self.env = copy(env) self.n_actions = n_actions
def __init__(self, name="chain"): """ Parameters - input_space: dimension of state vector """ self.sess = None self.log_prob = None self.default_dtype = get_default_tf_dtype() self.name = name self.omega_value = 0 self.width = 1 self.offset = 0 self.k = 1 self.param = 0.5
def __init__(self, state_space, action_space, hidden_layer_size, name="policy"): """ Builds a policy network and returns a node for the gradient and a node for action selection Simple network: from state space to action space Start from a random policy, all weights equal to 0 @param state_space: dimension of state space @param action_space: dimension of action space """ # net params super().__init__(name) self.hidden_layer_size = hidden_layer_size self.state_space = state_space self.action_space = action_space self.sess = None self.default_dtype = get_default_tf_dtype()
def __init__(self, model, policy, clip_gradient=False, env=None, n_trajectories=100): self.model = model self.policy = policy self.clip_gradient = clip_gradient self.dtype = get_default_tf_dtype() self.env = env self.global_step = 0 self.iteration = 0 self.n_trajectories = n_trajectories
def __init__(self, name="cartpole"): """ Parameters - input_space: dimension of state vector """ self.sess = None self.log_prob = None self.default_dtype = get_default_tf_dtype() # must be initialized self.name = name self.x_range = 4.8 self.theta_range = 180 # the noise (3sigma) should be inside 10% of the range self.x_var = 1e-6 # (self.x_range/(3*1000))**2 self.theta_var = 1e-6 # (self.theta_range/(3*1000))**2 self.x_dot_var = 1e-6 # self.x_var/1e-6 self.theta_dot_var = 1e-6 # self.theta_var/1e-6 self.action_noise_var = 1e-2 self.min_omega = 0.1 self.max_omega = 30
def __init__(self, state_space, action_space, hidden_layer_size, name="policy"): """ Builds a policy network and returns a node for the gradient and a node for action selection Simple network: from state space to action space Start from a random policy, all weights equal to 0 @param state_space: dimension of state space @param actions_space: dimension of action space @param trajectory_size: number of trajectories collected for estimating the gradient @param checkpoint_file: name of checkpoint file in which to save variables @param restore: True if need to restore variables """ # net params super().__init__(name) self.hidden_layer_size = hidden_layer_size self.state_space = state_space self.action_space = action_space self.sess = None self.default_dtype = get_default_tf_dtype()
def __init__(self, state_dim, param_dim, name="NN", training_set_size=4000): """ Fit a NN for predicting the next state distribution Output of NN are the parameters of a parametric distribution (Gaussian) """ self.sess = None self.log_prob = None self.prob = None self.dtype = get_default_tf_dtype() # must be initialized self.name = name self.x_range = 4.8 self.theta_range = 180 self.XData = None self.YData = None self.state_dim = state_dim self.x_dim = state_dim + param_dim self.gp_list = [] self.training_set_size = training_set_size self.global_step = 0 self.folder = self.name + "NNData" + "/" self.min_omega = 0.1 self.max_omega = 30
def __init__( self, kappa: float = 1e-3, L2_reg_dual: float = 0.0, # 1e-7,# 1e-5, L2_reg_loss: float = 0.0, max_opt_itr: int = 1000, tf_optimizer=ScipyOptimizerInterface, model: ModelApproximator = None, policy: Policy = None, env: ConfMDP = None, projection_type: Projection = Projection. STATE_KERNEL, # State kernel or disjoint training_set_size: int = 5000, exact: bool = False, restart_fitting: bool = False, fit_iterations: int = 40000, refit_iterations: int = 1000, refit: int = False, refit_every_iterations: int = 100, **kwargs, ): """ :param kappa: Max KL divergence between new policy and old policy. :param L2_reg_dual: Dual regularization :param L2_reg_loss: Loss regularization :param max_opt_itr: Maximum number of batch optimization iterations. :param tf_optimizer: optimizer to use :param model: model approximation :param policy: policy to be optimized :param env: environment :param projection_type: type of projection :param use_features: whether to use features or not :param training_set_size: number of samples in the training set :param exact: whether the model approximation is exact or not :return: """ self.kappa = kappa self.L2_reg_dual = L2_reg_dual self.L2_reg_loss = L2_reg_loss self.max_opt_itr = max_opt_itr self.tf_optimizer = tf_optimizer self.model = model self.policy = policy self.env = env self.dtype = get_default_tf_dtype() self.epsilon_small = 1e-24 self.min_eta_inv = 1e-12 self.projection_type = projection_type self.model_L2_reg_loss = 0 self.policy_L2_reg_loss = L2_reg_loss self.write_every = 1 self.training_set_size = training_set_size self.exact = exact self.fit_iterations = fit_iterations self.refit_iterations = (refit_iterations if not restart_fitting else fit_iterations) self.restart_fitting = restart_fitting self.refit = refit self.refit_every_iterations = refit_every_iterations self.sess = None self.summary_writer = None self.global_step = 0 self.iteration = 0 # ---------------------------------------- # placeholders # ---------------------------------------- self.observations_ph = None self.actions_one_hot_ph = None self.kappa_ph = None # Constraint on the KL divergence: \kappa self.actions_ph = None self.rewards_ph = None self.returns_ph = None self.timesteps_ph = None self.next_states_ph = None self.feat_diff_ph = None self.param_eta = None # Value of \eta self.param_eta_inv_ph = None # inverse of eta_ 1/\eta self.policy_tf = None # \pi(a | s) self.model_tf = None # p_\omega(s'|s,a) self.model_logli = None # log(p_\omega(s' | s,a)) self.model_policy_loss = None # self.dual = None # \min_{\eta\in[0, +\infty)} g(\eta) = # \eta \log \ev_{S,A,S' \sim d} \left[ \exp\left(\frac{1}{\eta} r(S,A,S') + \kappa \right) \right] self.dual_grad = None # Gradient of the dual self.primal = None # \exp\left(\frac{1}{\eta} r(s,a,s')\right) self.model_grad_loss = None # Gradient of the loss of the model self.policy_grad_loss = None # Gradient of the loss of the poliicy self.model_policy_grad_loss = None # Gradient of the loss of the model policy self.model_loss = None # Loss of the model self.policy_loss = None # Policy loss self.eta = None # Dual parameter self.state_kernel = ( None ) # p\{\theta, \omega} (s' | s) = p_\omega (s'|s,a) \pi_\theta (a | s) # ---------------------------------------- # Optimizers # ---------------------------------------- self.state_kernel_proj_opt = None self.model_tf_optimizer = None self.dual_optimizer = None # Summary self.summarize = None