def __init__(self, shape, mean=0.0, log_stddev=0.0, scope='gaussian', summary_labels=()): """ Categorical distribution. Args: shape: Action shape. mean: Optional distribution bias for the mean. log_stddev: Optional distribution bias for the standard deviation. """ self.shape = shape action_size = util.prod(self.shape) self.mean = Linear(size=action_size, bias=mean, scope='mean', summary_labels=summary_labels) self.log_stddev = Linear(size=action_size, bias=log_stddev, scope='log-stddev', summary_labels=summary_labels) super(Gaussian, self).__init__(shape=shape, scope=scope, summary_labels=summary_labels)
def __init__(self, shape, min_value, max_value, alpha=0.0, beta=0.0, scope='beta', summary_labels=()): """ Beta distribution. Args: shape: Action shape. min_value: Minimum value of continuous actions. max_value: Maximum value of continuous actions. alpha: Optional distribution bias for the alpha value. beta: Optional distribution bias for the beta value. """ assert min_value is None or max_value > min_value self.shape = shape self.min_value = min_value self.max_value = max_value action_size = util.prod(self.shape) self.alpha = Linear(size=action_size, bias=alpha, scope='alpha') self.beta = Linear(size=action_size, bias=beta, scope='beta') super(Beta, self).__init__(shape=shape, scope=scope, summary_labels=summary_labels)
def __init__(self, shape, min_value, max_value, alpha=0.0, beta=0.0, scope='beta', summary_labels=()): """ Beta distribution used for continuous actions. In particular, the Beta distribution allows to bound action values with min and max values. Args: shape: Shape of actions min_value: Min value of all actions for the given shape max_value: Max value of all actions for the given shape alpha: Concentration parameter of the Beta distribution beta: Concentration parameter of the Beta distribution """ assert min_value is None or max_value > min_value self.shape = shape self.min_value = min_value self.max_value = max_value action_size = util.prod(self.shape) self.alpha = Linear(size=action_size, bias=alpha, scope='alpha') self.beta = Linear(size=action_size, bias=beta, scope='beta') super(Beta, self).__init__(scope, summary_labels)
def initialize(self, custom_getter): super(QNAFModel, self).initialize(custom_getter) self.state_values = dict() self.l_entries = dict() for name, action in self.actions_spec.items(): num_action = util.prod(action['shape']) self.state_values[name] = Linear(size=num_action, scope='state-value') self.l_entries[name] = Linear(size=(num_action * (num_action - 1) // 2), scope='l-entries')
def setup_components_and_tf_funcs(self, custom_getter=None): super(QNAFModel, self).setup_components_and_tf_funcs(custom_getter) self.state_values = dict() self.l_entries = dict() for name, action in self.actions_spec.items(): num_action = util.prod(action['shape']) self.state_values[name] = Linear(size=num_action, scope='state-value') self.l_entries[name] = Linear(size=(num_action * (num_action - 1) // 2), scope='l-entries')
def __init__(self, shape, mean=0.0, log_stddev=0.0, scope='gaussian', summary_labels=()): self.shape = shape action_size = util.prod(self.shape) with tf.name_scope(name=scope): self.mean = Linear(size=action_size, bias=mean, scope='mean') self.log_stddev = Linear(size=action_size, bias=log_stddev, scope='log-stddev') super(Gaussian, self).__init__(scope, summary_labels)
def __init__(self, shape, num_actions, probabilities=None, scope='categorical', summary_labels=()): """ Categorical distribution. Args: shape: Action shape. num_actions: Number of discrete action alternatives. probabilities: Optional distribution bias. """ self.num_actions = num_actions action_size = util.prod(shape) * self.num_actions if probabilities is None: logits = 0.0 else: logits = [ log(prob) for _ in range(util.prod(shape)) for prob in probabilities ] self.logits = Linear(size=action_size, bias=logits, scope='logits') super(Categorical, self).__init__(shape=shape, scope=scope, summary_labels=summary_labels)
def __init__(self, shape, probability=0.5, scope='bernoulli', summary_labels=()): self.shape = shape action_size = util.prod(self.shape) with tf.name_scope(name=scope): self.logit = Linear(size=action_size, bias=log(probability), scope='logit') super(Bernoulli, self).__init__(scope, summary_labels)
def __init__(self, states_spec, actions_spec, network_spec, config): if any(action['type'] != 'float' or 'min_value' in action or 'max_value' in action for action in actions_spec.values()): raise TensorForceError("Only unconstrained float actions valid for NAFModel.") with tf.name_scope(name=config.scope): self.state_values = dict() self.l_entries = dict() for name, action in actions_spec.items(): num_action = util.prod(action['shape']) self.state_values[name] = Linear(size=num_action, scope=(name + 'state-value')) self.l_entries[name] = Linear(size=(num_action * (num_action - 1) // 2), scope=(name + '-l-entries')) super(QNAFModel, self).__init__( states_spec=states_spec, actions_spec=actions_spec, network_spec=network_spec, config=config )
def __init__(self, shape, num_actions, probabilities=None, scope='categorical', summary_labels=()): self.shape = shape self.num_actions = num_actions if probabilities is None: logits = 0.0 else: logits = [log(prob) for _ in range(util.prod(shape)) for prob in probabilities] action_size = util.prod(self.shape) * self.num_actions self.logits = Linear(size=action_size, bias=logits, scope='logits') super(Categorical, self).__init__(scope, summary_labels)
def __init__(self, network_spec, scope='network-baseline', summary_labels=()): """ Network baseline. Args: network_spec: Network specification dict """ with tf.name_scope(name=scope): self.network = Network.from_spec(spec=network_spec) assert len(self.network.internal_inputs()) == 0 self.linear = Linear(size=1, bias=0.0, scope='prediction') super(NetworkBaseline, self).__init__(scope, summary_labels)
def __init__(self, scope='ddpg-critic-network', summary_labels=(), size_t0=400, size_t1=300): super(DDPGCriticNetwork, self).__init__(scope=scope, summary_labels=summary_labels) self.t0l = Linear(size=size_t0, scope='linear0') self.t0b = TFLayer(layer='batch_normalization', scope='batchnorm0', center=True, scale=True) self.t0n = Nonlinearity(name='relu', scope='relu0') self.t1l = Linear(size=size_t1, scope='linear1') self.t1b = TFLayer(layer='batch_normalization', scope='batchnorm1', center=True, scale=True) self.t1n = Nonlinearity(name='relu', scope='relu1') self.t2d = Dense(size=1, activation='tanh', scope='dense0', weights=tf.random_uniform_initializer(minval=-3e-3, maxval=3e-3)) self.add_layer(self.t0l) self.add_layer(self.t0b) self.add_layer(self.t0n) self.add_layer(self.t1l) self.add_layer(self.t1b) self.add_layer(self.t1n) self.add_layer(self.t2d)
def __init__(self, network, scope='network-baseline', summary_labels=()): """ Network baseline. Args: network_spec: Network specification dict """ self.network = Network.from_spec( spec=network, kwargs=dict(summary_labels=summary_labels)) assert len(self.network.internals_spec()) == 0 self.linear = Linear(size=1, bias=0.0, scope='prediction') super(NetworkBaseline, self).__init__(scope=scope, summary_labels=summary_labels)
def __init__(self, baselines, scope='aggregated-baseline', summary_labels=()): """ Aggregated baseline. Args: baselines: Dict of per-state baseline specification dicts """ self.baselines = dict() for name in sorted(baselines): self.baselines[name] = Baseline.from_spec( spec=baselines[name], kwargs=dict(summary_labels=summary_labels)) self.linear = Linear(size=1, bias=0.0, scope='prediction', summary_labels=summary_labels) super(AggregatedBaseline, self).__init__(scope, summary_labels)
def __init__(self, baselines, scope='aggregated-baseline', summary_labels=()): """ Aggregated baseline. Args: baselines: Dict of per-state baseline specification dicts """ with tf.name_scope(name=scope): self.baselines = dict() for name, baseline_spec in baselines.items(): with tf.name_scope(name=(name + '-baseline')): self.baselines[name] = Baseline.from_spec( spec=baseline_spec, kwargs=dict(summary_labels=summary_labels) ) self.linear = Linear(size=1, bias=0.0, scope='prediction') super(AggregatedBaseline, self).__init__(scope, summary_labels)
def __init__(self, shape, probability=0.5, scope='bernoulli', summary_labels=()): """ Bernoulli distribution. Args: shape: Action shape. probability: Optional distribution bias. """ self.shape = shape action_size = util.prod(self.shape) self.logit = Linear(size=action_size, bias=log(probability), scope='logit') super(Bernoulli, self).__init__(shape=shape, scope=scope, summary_labels=summary_labels)