class DQFDAgent(MemoryAgent):
"""
Deep Q-learning from demonstration (DQFD) agent ([Hester et al., 2017](https://arxiv.org/abs/1704.03732)).
This agent uses DQN to pre-train from demonstration data.
Configuration:
Each agent requires the following configuration parameters:
* `states`: dict containing one or more state definitions.
* `actions`: dict containing one or more action definitions.
* `preprocessing`: dict or list containing state preprocessing configuration.
* `exploration`: dict containing action exploration configuration.
Each model requires the following configuration parameters:
* `discount`: float of discount factor (gamma).
* `learning_rate`: float of learning rate (alpha).
* `optimizer`: string of optimizer to use (e.g. 'adam').
* `device`: string of tensorflow device name.
* `tf_summary`: boolean indicating whether to use tensorflow summary file writer.
* `log_level`: string containing logleve (e.g. 'info').
* `distributed`: boolean indicating whether to use distributed tensorflow.
* `global_model`: global model.
* `session`: session to use.
The `DQFDAgent` class additionally requires the following parameters:
* `batch_size`: integer of the batch size.
* `memory_capacity`: integer of maximum experiences to store.
* `memory`: string indicating memory type ('replay' or 'prioritized_replay').
* `min_replay_size`: integer of minimum replay size before the first update.
* `update_rate`: float of the update rate (e.g. 0.25 = every 4 steps).
* `target_network_update_rate`: float of target network update rate (e.g. 0.01 = every 100 steps).
* `use_target_network`: boolean indicating whether to use a target network.
* `update_repeat`: integer of how many times to repeat an update.
* `update_target_weight`: float of update target weight (tau parameter).
* `demo_sampling_ratio`: float, ratio of expert data used at runtime to train from.
* `supervised_weight`: float, weight of large margin classifier loss.
* `expert_margin`: float of difference in Q-values between expert action and other actions enforced by the large margin function.
* `clip_loss`: float if not 0, uses the huber loss with clip_loss as the linear bound
"""
name = 'DQFDAgent'
model = DQFDModel
default_config = dict(target_update_frequency=10000,
demo_memory_capacity=1000000,
demo_sampling_ratio=0.01)
def __init__(self, config, model=None):
config.default(DQFDAgent.default_config)
super(DQFDAgent, self).__init__(config, model)
self.target_update_frequency = config.target_update_frequency
# This is the demonstration memory that we will fill with observations before starting
# the main training loop
self.demo_memory = Replay(config.demo_memory_capacity, config.states,
config.actions)
# The demo_sampling_ratio, called p in paper, controls ratio of expert vs online training samples
# p = n_demo / (n_demo + n_replay) => n_demo = p * n_replay / (1 - p)
self.demo_batch_size = int(config.demo_sampling_ratio *
config.batch_size /
(1.0 - config.demo_sampling_ratio))
assert self.demo_batch_size > 0, 'Check DQFD sampling parameters to make sure demo_batch_size is positive. (Calculated {} based on current parameters)'.format(
self.demo_batch_size)
def observe(self, reward, terminal):
"""Adds observations, updates via sampling from memories according to update rate.
DQFD samples from the online replay memory and the demo memory with
the fractions controlled by a hyper parameter p called 'expert sampling ratio.
Args:
reward:
terminal:
Returns:
"""
super(DQFDAgent, self).observe(reward=reward, terminal=terminal)
if self.timestep >= self.first_update and self.timestep % self.update_frequency == 0:
for _ in xrange(self.repeat_update):
batch = self.demo_memory.get_batch(self.demo_batch_size)
self.model.demonstration_update(batch=batch)
if self.timestep >= self.first_update and self.timestep % self.target_update_frequency == 0:
self.model.update_target()
def import_demonstrations(self, demonstrations):
"""Imports demonstrations, i.e. expert observations
Args:
demonstrations:
Returns:
"""
for observation in demonstrations:
if self.unique_state:
state = dict(state=observation['state'])
else:
state = observation['state']
if self.unique_action:
action = dict(action=observation['action'])
else:
action = observation['action']
self.demo_memory.add_observation(state=state,
action=action,
reward=observation['reward'],
terminal=observation['terminal'],
internal=observation['internal'])
def pretrain(self, steps):
"""Computes pretrain updates.
Args:
steps: Number of updates to execute.
Returns:
"""
for _ in xrange(steps):
# Sample from demo memory
batch = self.demo_memory.get_batch(batch_size=self.batch_size,
next_states=True)
# Update using both double Q-learning and supervised double_q_loss
self.model.demonstration_update(batch)
class PPOModel(PolicyGradientModel):
allows_discrete_actions = True
allows_continuous_actions = True
default_config = dict(
entropy_penalty=0.01,
loss_clipping=0.2, # Trust region clipping
epochs=10, # Number of training epochs for SGD,
optimizer_batch_size=128, # Batch size for optimiser
random_sampling=True # Sampling strategy for replay memory
)
def __init__(self, config):
config.default(PPOModel.default_config)
super(PPOModel, self).__init__(config)
self.optimizer_batch_size = config.optimizer_batch_size
# Use replay memory so memory logic can be used to sample batches
if self.optimizer_batch_size > config.batch_size:
raise Exception(
"optimizer_batch_size > batch_size ({}, {})".format(
self.optimizer_batch_size, config.batch_size))
self.updates = int(
config.batch_size / self.optimizer_batch_size) * config.epochs
self.memory = Replay(config.batch_size, config.states, config.actions,
config.random_sampling)
def create_tf_operations(self, config):
"""
Creates PPO training operations, i.e. the SGD update
based on the trust region loss.
:return:
"""
super(PPOModel, self).create_tf_operations(config)
with tf.variable_scope('update'):
prob_ratios = list()
entropy_penalties = list()
# for diagnostics
kl_divergences = list()
entropies = list()
self.distribution_tensors = dict()
self.prev_distribution_tensors = dict()
for name, action in self.action.items():
shape_size = util.prod(config.actions[name].shape)
distribution = self.distribution[name]
fixed_distribution = distribution.__class__.from_tensors(
tensors=[
tf.stop_gradient(x)
for x in distribution.get_tensors()
],
deterministic=self.deterministic)
# Standard policy gradient log likelihood computation
log_prob = distribution.log_probability(action=action)
fixed_log_prob = fixed_distribution.log_probability(
action=action)
log_prob_diff = log_prob - fixed_log_prob
prob_ratio = tf.exp(x=log_prob_diff)
prob_ratio = tf.reshape(tensor=prob_ratio,
shape=(-1, shape_size))
prob_ratios.append(prob_ratio)
entropy = distribution.entropy()
entropy_penalty = -config.entropy_penalty * entropy
entropy_penalty = tf.reshape(tensor=entropy_penalty,
shape=(-1, shape_size))
entropy_penalties.append(entropy_penalty)
self.distribution_tensors[name] = list(
distribution.get_tensors())
prev_distribution = list(
tf.placeholder(dtype=tf.float32,
shape=util.shape(tensor, unknown=None))
for tensor in distribution.get_tensors())
self.prev_distribution_tensors[name] = prev_distribution
prev_distribution = distribution.from_tensors(
tensors=prev_distribution,
deterministic=self.deterministic)
kl_divergence = prev_distribution.kl_divergence(
other=distribution)
kl_divergence = tf.reshape(tensor=kl_divergence,
shape=(-1, shape_size))
kl_divergences.append(kl_divergence)
entropy = tf.reshape(tensor=entropy, shape=(-1, shape_size))
entropies.append(entropy)
# The surrogate loss in PPO is the minimum of clipped loss and
# target advantage * prob_ratio, which is the CPO loss
# Presentation on conservative policy iteration:
# https://www.cs.cmu.edu/~jcl/presentation/RL/RL.ps
prob_ratio = tf.reduce_mean(input_tensor=tf.concat(
values=prob_ratios, axis=1),
axis=1)
tf.summary.histogram('prob_ratio', prob_ratio)
tf.summary.scalar('mean_prob_ratio',
tf.reduce_mean(input_tensor=prob_ratio, axis=0))
clipped_prob_ratio = tf.clip_by_value(prob_ratio,
1.0 - config.loss_clipping,
1.0 + config.loss_clipping)
self.loss_per_instance = -tf.minimum(
x=(prob_ratio * self.reward),
y=(clipped_prob_ratio * self.reward))
self.surrogate_loss = tf.reduce_mean(
input_tensor=self.loss_per_instance,
axis=0,
name='surrogate_loss')
tf.losses.add_loss(self.surrogate_loss)
# Mean over actions, mean over batch
entropy_penalty = tf.reduce_mean(input_tensor=tf.concat(
values=entropy_penalties, axis=1),
axis=1)
self.entropy_penalty = tf.reduce_mean(input_tensor=entropy_penalty,
axis=0,
name='entropy_penalty')
tf.losses.add_loss(self.entropy_penalty)
kl_divergence = tf.reduce_mean(input_tensor=tf.concat(
values=kl_divergences, axis=1),
axis=1)
self.kl_divergence = tf.reduce_mean(input_tensor=kl_divergence,
axis=0)
tf.summary.scalar('kl_divergence', self.kl_divergence)
entropy = tf.reduce_mean(input_tensor=tf.concat(values=entropies,
axis=1),
axis=1)
self.entropy = tf.reduce_mean(input_tensor=entropy, axis=0)
tf.summary.scalar('entropy', self.entropy)
def update(self, batch):
"""
Compute update for one batch of experiences using general advantage estimation
and the trust region update based on SGD on the clipped loss.
:param batch: On policy batch of experiences.
:return:
"""
batch['rewards'], discounted_rewards = self.reward_estimation(
states=batch['states'],
rewards=batch['rewards'],
terminals=batch['terminals'])
if self.baseline:
self.baseline.update(states=batch['states'],
returns=discounted_rewards)
# Set memory contents to batch contents
self.memory.set_memory(states=batch['states'],
actions=batch['actions'],
rewards=batch['rewards'],
terminals=batch['terminals'],
internals=batch['internals'])
# PPO takes multiple passes over the on-policy batch.
# We use a memory sampling random ranges (as opposed to keeping
# track of indices and e.g. first taking elems 0-15, then 16-32, etc).
for i in xrange(self.updates):
self.logger.debug('Optimising PPO, update = {}'.format(i))
batch = self.memory.get_batch(self.optimizer_batch_size)
feed_dict = {
state: batch['states'][name]
for name, state in self.state.items()
}
feed_dict.update({
action: batch['actions'][name]
for name, action in self.action.items()
})
feed_dict[self.reward] = batch['rewards']
feed_dict[self.terminal] = batch['terminals']
feed_dict.update({
internal: batch['internals'][n]
for n, internal in enumerate(self.internal_inputs)
})
if i == 0: # First update, fetch previous distribution tensors
assert self.updates >= 2
assert 'optimize' not in self.distribution_tensors
fetches = dict(optimize=self.optimize)
fetches.update(self.distribution_tensors)
prev_distribution_tensors = self.session.run(
fetches=fetches, feed_dict=feed_dict)
prev_distribution_tensors.pop('optimize')
elif i == self.updates - 1: # Last update, fetch return and diagnostics values
fetches = [
self.optimize, self.loss, self.loss_per_instance,
self.kl_divergence, self.entropy
]
prev_distribution_tensors = {
placeholder: tensor
for name, placeholders in
self.prev_distribution_tensors.items()
for placeholder, tensor in zip(
placeholders, prev_distribution_tensors[name])
}
feed_dict.update(prev_distribution_tensors)
with SummarySessionWrapper(self, fetches,
feed_dict) as session:
_, loss, loss_per_instance, kl_divergence, entropy = session.run(
)
else: # Otherwise just optimize
self.session.run(fetches=self.optimize, feed_dict=feed_dict)
return loss, loss_per_instance
class DQFDAgent(MemoryAgent):
"""
Deep Q-learning from demonstration (DQFD) agent ([Hester et al., 2017](https://arxiv.org/abs/1704.03732)).
This agent uses DQN to pre-train from demonstration data.
Configuration:
Each agent requires the following configuration parameters:
* `states`: dict containing one or more state definitions.
* `actions`: dict containing one or more action definitions.
* `preprocessing`: dict or list containing state preprocessing configuration.
* `exploration`: dict containing action exploration configuration.
Each model requires the following configuration parameters:
* `discount`: float of discount factor (gamma).
* `learning_rate`: float of learning rate (alpha).
* `optimizer`: string of optimizer to use (e.g. 'adam').
* `device`: string of tensorflow device name.
* `tf_summary`: string directory to write tensorflow summaries. Default None
* `tf_summary_level`: int indicating which tensorflow summaries to create.
* `tf_summary_interval`: int number of calls to get_action until writing tensorflow summaries on update.
* `log_level`: string containing logleve (e.g. 'info').
* `distributed`: boolean indicating whether to use distributed tensorflow.
* `global_model`: global model.
* `session`: session to use.
The `DQFDAgent` class additionally requires the following parameters:
* `batch_size`: integer of the batch size.
* `memory_capacity`: integer of maximum experiences to store.
* `memory`: string indicating memory type ('replay' or 'prioritized_replay').
* `min_replay_size`: integer of minimum replay size before the first update.
* `update_rate`: float of the update rate (e.g. 0.25 = every 4 steps).
* `target_network_update_rate`: float of target network update rate (e.g. 0.01 = every 100 steps).
* `use_target_network`: boolean indicating whether to use a target network.
* `update_repeat`: integer of how many times to repeat an update.
* `update_target_weight`: float of update target weight (tau parameter).
* `demo_sampling_ratio`: float, ratio of expert data used at runtime to train from.
* `supervised_weight`: float, weight of large margin classifier loss.
* `expert_margin`: float of difference in Q-values between expert action and other actions enforced
by the large margin function.
* `clip_loss`: float if not 0, uses the huber loss with clip_loss as the linear bound
"""
default_config = dict(
# Agent
preprocessing=None,
exploration=None,
reward_preprocessing=None,
# Model
optimizer=dict(
type='adam',
learning_rate=1e-3
),
discount=0.99,
normalize_rewards=False,
variable_noise=None, # not documented!!!
# DistributionModel
distributions=None, # not documented!!!
entropy_regularization=None,
# QModel
target_sync_frequency=10000, # not documented!!!
target_update_weight=1.0, # not documented!!!
huber_loss=0.0, # not documented!!!
# Logging
log_level='info',
model_directory=None,
save_frequency=600, # TensorFlow default
summary_labels=['total-loss'],
summary_frequency=120, # TensorFlow default
# TensorFlow distributed configuration
cluster_spec=None,
parameter_server=False,
task_index=0,
device=None,
local_model=False,
replica_model=False,
scope='dqfd'
)
def __init__(self, states_spec, actions_spec, network_spec, config):
self.network_spec = network_spec
config = config.copy()
config.default(DQFDAgent.default_config)
# DQFD always uses double dqn, which is a required key for a q-model.
config.obligatory(double_dqn=True)
self.target_update_frequency = config.target_update_frequency
self.demo_memory_capacity = config.demo_memory_capacity
# The demo_sampling_ratio, called p in paper, controls ratio of expert vs online training samples
# p = n_demo / (n_demo + n_replay) => n_demo = p * n_replay / (1 - p)
self.demo_batch_size = int(config.demo_sampling_ratio * config.batch_size / (1.0 - config.demo_sampling_ratio))
assert self.demo_batch_size > 0, 'Check DQFD sampling parameters to ensure ' \
'demo_batch_size is positive. (Calculated {} based on current' \
' parameters)'.format(self.demo_batch_size)
# This is the demonstration memory that we will fill with observations before starting
# the main training loop
self.demo_memory = Replay(self.demo_memory_capacity, self.states_spec, self.actions_spec)
super(DQFDAgent, self).__init__(
states_spec=states_spec,
actions_spec=actions_spec,
config=config
)
def observe(self, reward, terminal):
"""
Adds observations, updates via sampling from memories according to update rate.
DQFD samples from the online replay memory and the demo memory with
the fractions controlled by a hyper parameter p called 'expert sampling ratio.
Args:
reward:
terminal:
"""
super(DQFDAgent, self).observe(reward=reward, terminal=terminal)
if self.timestep >= self.first_update and self.timestep % self.update_frequency == 0:
for _ in xrange(self.repeat_update):
batch = self.demo_memory.get_batch(self.demo_batch_size)
self.model.demonstration_update(batch=batch)
def import_demonstrations(self, demonstrations):
"""
Imports demonstrations, i.e. expert observations. Note that for large numbers of observations,
set_demonstrations is more appropriate, which directly sets memory contents to an array an expects
a different layout.
Args:
demonstrations: List of observation dicts
"""
for observation in demonstrations:
if self.unique_state:
state = dict(state=observation['states'])
else:
state = observation['states']
if self.unique_action:
action = dict(action=observation['actions'])
else:
action = observation['actions']
self.demo_memory.add_observation(
states=state,
internals=observation['internal'],
actions=action,
terminal=observation['terminal'],
reward=observation['reward']
)
def set_demonstrations(self, batch):
"""
Set all demonstrations from batch data. Expects a dict wherein each value contains an array
containing all states, actions, rewards, terminals and internals respectively.
Args:
batch:
"""
self.demo_memory.set_memory(
states=batch['states'],
internals=batch['internals'],
actions=batch['actions'],
terminal=batch['terminal'],
reward=batch['reward']
)
def initialize_model(self, states_spec, actions_spec, config):
return QDemoModel(
states_spec=states_spec,
actions_spec=actions_spec,
network_spec=self.network_spec,
config=config
)
def pretrain(self, steps):
"""
Computes pretrain updates.
Args:
steps: Number of updates to execute.
"""
for _ in xrange(steps):
# Sample from demo memory.
batch = self.demo_memory.get_batch(batch_size=self.batch_size, next_states=True)
# Update using both double Q-learning and supervised double_q_loss.
self.model.demonstration_update(batch)
class DQFDAgent(MemoryAgent):
"""
Deep Q-learning from demonstration (DQFD) agent ([Hester et al., 2017](https://arxiv.org/abs/1704.03732)).
This agent uses DQN to pre-train from demonstration data.
Configuration:
Each agent requires the following configuration parameters:
* `states`: dict containing one or more state definitions.
* `actions`: dict containing one or more action definitions.
* `preprocessing`: dict or list containing state preprocessing configuration.
* `exploration`: dict containing action exploration configuration.
Each model requires the following configuration parameters:
* `discount`: float of discount factor (gamma).
* `learning_rate`: float of learning rate (alpha).
* `optimizer`: string of optimizer to use (e.g. 'adam').
* `device`: string of tensorflow device name.
* `tf_summary`: string directory to write tensorflow summaries. Default None
* `tf_summary_level`: int indicating which tensorflow summaries to create.
* `tf_summary_interval`: int number of calls to get_action until writing tensorflow summaries on update.
* `log_level`: string containing logleve (e.g. 'info').
* `distributed`: boolean indicating whether to use distributed tensorflow.
* `global_model`: global model.
* `session`: session to use.
The `DQFDAgent` class additionally requires the following parameters:
* `batch_size`: integer of the batch size.
* `memory_capacity`: integer of maximum experiences to store.
* `memory`: string indicating memory type ('replay' or 'prioritized_replay').
* `min_replay_size`: integer of minimum replay size before the first update.
* `update_rate`: float of the update rate (e.g. 0.25 = every 4 steps).
* `target_network_update_rate`: float of target network update rate (e.g. 0.01 = every 100 steps).
* `use_target_network`: boolean indicating whether to use a target network.
* `update_repeat`: integer of how many times to repeat an update.
* `update_target_weight`: float of update target weight (tau parameter).
* `demo_sampling_ratio`: float, ratio of expert data used at runtime to train from.
* `supervised_weight`: float, weight of large margin classifier loss.
* `expert_margin`: float of difference in Q-values between expert action and other actions enforced
by the large margin function.
* `clip_loss`: float if not 0, uses the huber loss with clip_loss as the linear bound
"""
def __init__(self,
states_spec,
actions_spec,
network_spec,
device=None,
scope='dqfd',
saver_spec=None,
summary_spec=None,
distributed_spec=None,
optimizer=None,
discount=0.99,
normalize_rewards=False,
variable_noise=None,
distributions_spec=None,
entropy_regularization=None,
target_sync_frequency=10000,
target_update_weight=1.0,
huber_loss=None,
preprocessing=None,
exploration=None,
reward_preprocessing=None,
batched_observe=1000,
batch_size=32,
memory=None,
first_update=10000,
update_frequency=4,
repeat_update=1,
expert_margin=0.5,
supervised_weight=0.1,
demo_memory_capacity=10000,
demo_sampling_ratio=0.2):
"""
Deep Q-learning from demonstration (DQFD) agent ([Hester et al., 2017](https://arxiv.org/abs/1704.03732)).
This agent uses DQN to pre-train from demonstration data in combination with a supervised loss.
Args:
states_spec:
actions_spec:
network_spec:
device:
scope:
saver_spec:
summary_spec:
distributed_spec:
optimizer:
discount:
normalize_rewards:
variable_noise:
distributions_spec:
entropy_regularization:
target_sync_frequency:
target_update_weight:
double_q_model:
huber_loss:
preprocessing:
exploration:
reward_preprocessing:
batched_observe:
batch_size:
memory:
first_update:
update_frequency:
repeat_update:
expert_margin:
supervised_weight:
demo_memory_capacity:
demo_sampling_ratio:
"""
if network_spec is None:
raise TensorForceError("No network_spec provided.")
if optimizer is None:
self.optimizer = dict(type='adam', learning_rate=1e-3)
else:
self.optimizer = optimizer
if memory is None:
memory = dict(type='replay', capacity=100000)
else:
self.memory = memory
self.network_spec = network_spec
self.device = device
self.scope = scope
self.saver_spec = saver_spec
self.summary_spec = summary_spec
self.distributed_spec = distributed_spec
self.discount = discount
self.normalize_rewards = normalize_rewards
self.variable_noise = variable_noise
self.distributions_spec = distributions_spec
self.entropy_regularization = entropy_regularization
self.target_sync_frequency = target_sync_frequency
self.target_update_weight = target_update_weight
self.double_q_model = double_q_model
self.huber_loss = huber_loss
# DQFD always uses double dqn, which is a required key for a q-model.
self.double_q_model = True
self.target_sync_frequency = target_sync_frequency
self.demo_memory_capacity = demo_memory_capacity
self.expert_margin = expert_margin
self.supervised_weight = supervised_weight
# The demo_sampling_ratio, called p in paper, controls ratio of expert vs online training samples
# p = n_demo / (n_demo + n_replay) => n_demo = p * n_replay / (1 - p)
self.demo_batch_size = int(demo_sampling_ratio * batch_size /
(1.0 - demo_sampling_ratio))
assert self.demo_batch_size > 0, 'Check DQFD sampling parameters to ensure ' \
'demo_batch_size is positive. (Calculated {} based on current' \
' parameters)'.format(self.demo_batch_size)
# This is the demonstration memory that we will fill with observations before starting
# the main training loop
super(DQFDAgent,
self).__init__(states_spec=states_spec,
actions_spec=actions_spec,
preprocessing=preprocessing,
exploration=exploration,
reward_preprocessing=reward_preprocessing,
batched_observe=batched_observe,
batch_size=batch_size,
memory=memory,
first_update=first_update,
update_frequency=update_frequency,
repeat_update=repeat_update)
self.demo_memory = Replay(self.states_spec, self.actions_spec,
self.demo_memory_capacity)
def initialize_model(self, states_spec, actions_spec):
return QDemoModel(
states_spec=states_spec,
actions_spec=actions_spec,
network_spec=self.network_spec,
device=self.device,
scope=self.scope,
saver_spec=self.saver_spec,
summary_spec=self.summary_spec,
distributed_spec=self.distributed_spec,
optimizer=self.optimizer,
discount=self.discount,
normalize_rewards=self.normalize_rewards,
variable_noise=self.variable_noise,
distributions_spec=self.distributions_spec,
entropy_regularization=self.entropy_regularization,
target_sync_frequency=self.target_sync_frequency,
target_update_weight=self.target_update_weight,
double_q_model=self.double_q_model,
huber_loss=self.huber_loss,
# TEMP: Random sampling fix
random_sampling_fix=True,
expert_margin=self.expert_margin,
supervised_weight=self.supervised_weight)
def observe(self, reward, terminal):
"""
Adds observations, updates via sampling from memories according to update rate.
DQFD samples from the online replay memory and the demo memory with
the fractions controlled by a hyper parameter p called 'expert sampling ratio.
Args:
reward:
terminal:
"""
super(DQFDAgent, self).observe(reward=reward, terminal=terminal)
if self.timestep >= self.first_update and self.timestep % self.update_frequency == 0:
for _ in xrange(self.repeat_update):
batch = self.demo_memory.get_batch(self.demo_batch_size)
self.model.demonstration_update(batch=batch)
def import_demonstrations(self, demonstrations):
"""
Imports demonstrations, i.e. expert observations. Note that for large numbers of observations,
set_demonstrations is more appropriate, which directly sets memory contents to an array an expects
a different layout.
Args:
demonstrations: List of observation dicts
"""
for observation in demonstrations:
if self.unique_state:
state = dict(state=observation['states'])
else:
state = observation['states']
if self.unique_action:
action = dict(action=observation['actions'])
else:
action = observation['actions']
self.demo_memory.add_observation(
states=state,
internals=observation['internals'],
actions=action,
terminal=observation['terminal'],
reward=observation['reward'])
def set_demonstrations(self, batch):
"""
Set all demonstrations from batch data. Expects a dict wherein each value contains an array
containing all states, actions, rewards, terminals and internals respectively.
Args:
batch:
"""
self.demo_memory.set_memory(states=batch['states'],
internals=batch['internals'],
actions=batch['actions'],
terminal=batch['terminal'],
reward=batch['reward'])
def pretrain(self, steps):
"""
Computes pretrain updates.
Args:
steps: Number of updates to execute.
"""
for _ in xrange(steps):
# Sample from demo memory.
batch = self.demo_memory.get_batch(batch_size=self.batch_size,
next_states=True)
# Update using both double Q-learning and supervised double_q_loss.
self.model.demonstration_update(batch)
class DQFDAgent(MemoryAgent):
"""
Deep Q-learning from demonstration (DQFD) agent ([Hester et al., 2017](https://arxiv.org/abs/1704.03732)).
This agent uses DQN to pre-train from demonstration data.
Configuration:
Each agent requires the following configuration parameters:
* `states`: dict containing one or more state definitions.
* `actions`: dict containing one or more action definitions.
* `preprocessing`: dict or list containing state preprocessing configuration.
* `exploration`: dict containing action exploration configuration.
Each model requires the following configuration parameters:
* `discount`: float of discount factor (gamma).
* `learning_rate`: float of learning rate (alpha).
* `optimizer`: string of optimizer to use (e.g. 'adam').
* `optimizer_args`: list of arguments for optimizer.
* `optimizer_kwargs`: dict of keyword arguments for optimizer.
* `device`: string of tensorflow device name.
* `tf_saver`: boolean whether to save model parameters.
* `tf_summary`: boolean indicating whether to use tensorflow summary file writer.
* `log_level`: string containing logleve (e.g. 'info').
* `distributed`: boolean indicating whether to use distributed tensorflow.
* `global_model`: global model.
* `session`: session to use.
The `DQFDAgent` class additionally requires the following parameters:
* `batch_size`: integer of the batch size.
* `memory_capacity`: integer of maximum experiences to store.
* `memory`: string indicating memory type ('replay' or 'prioritized_replay').
* `memory_args`: list of arguments to pass to replay memory constructor.
* `memory_kwargs`: list of keyword arguments to pass to replay memory constructor.
* `min_replay_size`: integer of minimum replay size before the first update.
* `update_rate`: float of the update rate (e.g. 0.25 = every 4 steps).
* `target_network_update_rate`: float of target network update rate (e.g. 0.01 = every 100 steps).
* `use_target_network`: boolean indicating whether to use a target network.
* `update_repeat`: integer of how many times to repeat an update.
* `update_target_weight`: float of update target weight (tau parameter).
* `demo_sampling_ratio`: float, ratio of expert data used at runtime to train from.
* `supervised_weight`: float, weight of large margin classifier loss.
* `expert_margin`: float of difference in Q-values between expert action and other actions enforced by the large margin function.
* `clip_gradients`: float of maximum values for gradients before clipping.
"""
name = 'DQFDAgent'
model = DQFDModel
default_config = dict(
target_update_frequency=10000,
demo_memory_capacity=1000000,
demo_sampling_ratio=0.01
)
def __init__(self, config):
config.default(DQFDAgent.default_config)
super(DQFDAgent, self).__init__(config)
self.target_update_frequency = config.target_update_frequency
# This is the demonstration memory that we will fill with observations before starting
# the main training loop
self.demo_memory = Replay(config.demo_memory_capacity, config.states, config.actions)
# The demo_sampling_ratio, called p in paper, controls ratio of expert vs online training samples
# p = n_demo / (n_demo + n_replay) => n_demo = p * n_replay / (1 - p)
self.demo_batch_size = int(config.demo_sampling_ratio * config.batch_size / (1.0 - config.demo_sampling_ratio))
assert self.demo_batch_size > 0, 'Check DQFD sampling parameters to make sure demo_batch_size is positive.' \
'(Calculated {} based on current parameters)'.format(self.demo_batch_size)
def observe(self, reward, terminal):
"""Adds observations, updates via sampling from memories according to update rate.
DQFD samples from the online replay memory and the demo memory with
the fractions controlled by a hyper parameter p called 'expert sampling ratio.
Args:
reward:
terminal:
Returns:
"""
super(DQFDAgent, self).observe(reward=reward, terminal=terminal)
if self.timestep >= self.first_update and self.timestep % self.update_frequency == 0:
for _ in xrange(self.repeat_update):
batch = self.demo_memory.get_batch(self.demo_batch_size)
self.model.demonstration_update(batch=batch)
if self.timestep >= self.first_update and self.timestep % self.target_update_frequency == 0:
self.model.update_target()
def import_demonstrations(self, demonstrations):
"""Imports demonstrations, i.e. expert observations
Args:
demonstrations:
Returns:
"""
for observation in demonstrations:
if self.unique_state:
state = dict(state=observation['state'])
else:
state = observation['state']
if self.unique_action:
action = dict(action=observation['action'])
else:
action = observation['action']
self.demo_memory.add_observation(
state=state,
action=action,
reward=observation['reward'],
terminal=observation['terminal'],
internal=observation['internal']
)
def pretrain(self, steps):
"""Computes pretrain updates.
Args:
steps: Number of updates to execute.
Returns:
"""
for _ in xrange(steps):
# Sample from demo memory
batch = self.demo_memory.get_batch(self.batch_size)
# Update using both double Q-learning and supervised double_q_loss
self.model.demonstration_update(batch)
class PPOModel(PolicyGradientModel):
allows_discrete_actions = True
allows_continuous_actions = True
default_config = dict(
entropy_penalty=0.01,
loss_clipping=0.1, # Trust region clipping
epochs=10, # Number of training epochs for SGD,
optimizer_batch_size=128, # Batch size for optimiser
random_sampling=True # Sampling strategy for replay memory
)
def __init__(self, config):
config.default(PPOModel.default_config)
super(PPOModel, self).__init__(config)
self.optimizer_batch_size = config.optimizer_batch_size
# Use replay memory so memory logic can be used to sample batches
self.updates = int(
config.batch_size / self.optimizer_batch_size) * config.epochs
self.memory = Replay(config.batch_size, config.states, config.actions,
config.random_sampling)
def create_tf_operations(self, config):
"""
Creates PPO training operations, i.e. the SGD update
based on the trust region loss.
:return:
"""
super(PPOModel, self).create_tf_operations(config)
with tf.variable_scope('update'):
prob_ratios = list()
entropy_penalties = list()
kl_divergences = list()
entropies = list()
for name, action in self.action.items():
distribution = self.distribution[name]
prev_distribution = tuple(
tf.placeholder(dtype=tf.float32,
shape=util.shape(x, unknown=None))
for x in distribution)
self.internal_inputs.extend(prev_distribution)
self.internal_outputs.extend(distribution)
self.internal_inits.extend(
np.zeros(shape=util.shape(x)[1:]) for x in distribution)
prev_distribution = self.distribution[
name].__class__.from_tensors(
parameters=prev_distribution,
deterministic=self.deterministic)
shape_size = util.prod(config.actions[name].shape)
# Standard policy gradient log likelihood computation
log_prob = distribution.log_probability(action=action)
prev_log_prob = prev_distribution.log_probability(
action=action)
log_prob_diff = tf.minimum(x=(log_prob - prev_log_prob),
y=10.0)
prob_ratio = tf.exp(x=log_prob_diff)
prob_ratio = tf.reshape(tensor=prob_ratio,
shape=(-1, shape_size))
prob_ratios.append(prob_ratio)
entropy = distribution.entropy()
entropy_penalty = -config.entropy_penalty * entropy
entropy_penalty = tf.reshape(tensor=entropy_penalty,
shape=(-1, shape_size))
entropy_penalties.append(entropy_penalty)
entropy = tf.reshape(tensor=entropy, shape=(-1, shape_size))
entropies.append(entropy)
kl_divergence = distribution.kl_divergence(prev_distribution)
kl_divergence = tf.reshape(tensor=kl_divergence,
shape=(-1, shape_size))
kl_divergences.append(kl_divergence)
# The surrogate loss in PPO is the minimum of clipped loss and
# target advantage * prob_ratio, which is the CPO loss
# Presentation on conservative policy iteration:
# https://www.cs.cmu.edu/~jcl/presentation/RL/RL.ps
prob_ratio = tf.reduce_mean(input_tensor=tf.concat(
values=prob_ratios, axis=1),
axis=1)
prob_ratio = tf.clip_by_value(prob_ratio,
1.0 - config.loss_clipping,
1.0 + config.loss_clipping)
self.loss_per_instance = -prob_ratio * self.reward
self.surrogate_loss = tf.reduce_mean(
input_tensor=self.loss_per_instance, axis=0)
tf.losses.add_loss(self.surrogate_loss)
# Mean over actions, mean over batch
entropy_penalty = tf.reduce_mean(input_tensor=tf.concat(
values=entropy_penalties, axis=1),
axis=1)
self.entropy_penalty = tf.reduce_mean(input_tensor=entropy_penalty,
axis=0)
tf.losses.add_loss(self.entropy_penalty)
entropy = tf.reduce_mean(input_tensor=tf.concat(values=entropies,
axis=1),
axis=1)
self.entropy = tf.reduce_mean(input_tensor=entropy, axis=0)
kl_divergence = tf.reduce_mean(input_tensor=tf.concat(
values=kl_divergences, axis=1),
axis=1)
self.kl_divergence = tf.reduce_mean(input_tensor=kl_divergence,
axis=0)
def update(self, batch):
"""
Compute update for one batch of experiences using general advantage estimation
and the trust region update based on SGD on the clipped loss.
:param batch: On policy batch of experiences.
:return:
"""
# Compute GAE.
self.advantage_estimation(batch)
if self.baseline:
self.baseline.update(states=batch['states'],
returns=batch['returns'])
# Set memory contents to batch contents
self.memory.set_memory(states=batch['states'],
actions=batch['actions'],
rewards=batch['rewards'],
terminals=batch['terminals'],
internals=batch['internals'])
# PPO takes multiple passes over the on-policy batch.
# We use a memory sampling random ranges (as opposed to keeping
# track of indices and e.g. first taking elems 0-15, then 16-32, etc).
for i in xrange(self.updates):
self.logger.debug('Optimising PPO, update = {}'.format(i))
batch = self.memory.get_batch(self.optimizer_batch_size)
fetches = [
self.optimize, self.loss, self.loss_per_instance,
self.kl_divergence, self.entropy
]
feed_dict = {
state: batch['states'][name]
for name, state in self.state.items()
}
feed_dict.update({
action: batch['actions'][name]
for name, action in self.action.items()
})
feed_dict[self.reward] = batch['rewards']
feed_dict[self.terminal] = batch['terminals']
feed_dict.update({
internal: batch['internals'][n]
for n, internal in enumerate(self.internal_inputs)
})
loss, loss_per_instance, kl_divergence, entropy = self.session.run(
fetches=fetches, feed_dict=feed_dict)[1:5]
self.logger.debug('Loss = {}'.format(loss))
self.logger.debug('KL divergence = {}'.format(kl_divergence))
self.logger.debug('Entropy = {}'.format(entropy))
return loss, loss_per_instance
class DQFDAgent(MemoryAgent):
"""
Deep Q-learning from demonstration (DQFD) agent ([Hester et al., 2017](https://arxiv.org/abs/1704.03732)).
This agent uses DQN to pre-train from demonstration data via an additional supervised loss term.
"""
def __init__(self,
states_spec,
actions_spec,
network_spec,
device=None,
session_config=None,
scope='dqfd',
saver_spec=None,
summary_spec=None,
distributed_spec=None,
optimizer=None,
discount=0.99,
variable_noise=None,
states_preprocessing_spec=None,
explorations_spec=None,
reward_preprocessing_spec=None,
distributions_spec=None,
entropy_regularization=None,
target_sync_frequency=10000,
target_update_weight=1.0,
huber_loss=None,
batched_observe=1000,
batch_size=32,
memory=None,
first_update=10000,
update_frequency=4,
repeat_update=1,
expert_margin=0.5,
supervised_weight=0.1,
demo_memory_capacity=10000,
demo_sampling_ratio=0.2):
"""
Deep Q-learning from demonstration (DQFD) agent ([Hester et al., 2017](https://arxiv.org/abs/1704.03732)).
This agent uses DQN to pre-train from demonstration data in combination with a supervised loss.
Args:
states_spec: Dict containing at least one state definition. In the case of a single state,
keys `shape` and `type` are necessary. For multiple states, pass a dict of dicts where each state
is a dict itself with a unique name as its key.
actions_spec: Dict containing at least one action definition. Actions have types and either `num_actions`
for discrete actions or a `shape` for continuous actions. Consult documentation and tests for more.
network_spec: List of layers specifying a neural network via layer types, sizes and optional arguments
such as activation or regularisation. Full examples are in the examples/configs folder.
device: Device string specifying model device.
session_config: optional tf.ConfigProto with additional desired session configurations
scope: TensorFlow scope, defaults to agent name (e.g. `dqn`).
saver_spec: Dict specifying automated saving. Use `directory` to specify where checkpoints are saved. Use
either `seconds` or `steps` to specify how often the model should be saved. The `load` flag specifies
if a model is initially loaded (set to True) from a file `file`.
summary_spec: Dict specifying summaries for TensorBoard. Requires a 'directory' to store summaries, `steps`
or `seconds` to specify how often to save summaries, and a list of `labels` to indicate which values
to export, e.g. `losses`, `variables`. Consult neural network class and model for all available labels.
distributed_spec: Dict specifying distributed functionality. Use `parameter_server` and `replica_model`
Boolean flags to indicate workers and parameter servers. Use a `cluster_spec` key to pass a TensorFlow
cluster spec.
optimizer: Dict specifying optimizer type and its optional parameters, typically a `learning_rate`.
Available optimizer types include standard TensorFlow optimizers, `natural_gradient`,
and `evolutionary`. Consult the optimizer test or example configurations for more.
discount: Float specifying reward discount factor.
variable_noise: Experimental optional parameter specifying variable noise (NoisyNet).
states_preprocessing_spec: Optional list of states preprocessors to apply to state
(e.g. `image_resize`, `grayscale`).
explorations_spec: Optional dict specifying action exploration type (epsilon greedy
or Gaussian noise).
reward_preprocessing_spec: Optional dict specifying reward preprocessing.
distributions_spec: Optional dict specifying action distributions to override default distribution choices.
Must match action names.
entropy_regularization: Optional positive float specifying an entropy regularization value.
target_sync_frequency: Interval between optimization calls synchronizing the target network.
target_update_weight: Update weight, 1.0 meaning a full assignment to target network from training network.
huber_loss: Optional flat specifying Huber-loss clipping.
batched_observe: Optional int specifying how many observe calls are batched into one session run.
Without batching, throughput will be lower because every `observe` triggers a session invocation to
update rewards in the graph.
batch_size: Int specifying batch size used to sample from memory. Should be smaller than memory size.
memory: Dict describing memory via `type` (e.g. `replay`) and `capacity`.
first_update: Int describing at which time step the first update is performed. Should be larger
than batch size.
update_frequency: Int specifying number of observe steps to perform until an update is executed.
repeat_update: Int specifying how many update steps are performed per update, where each update step implies
sampling a batch from the memory and passing it to the model.
expert_margin: Positive float specifying enforced supervised margin between expert action Q-value and other
Q-values.
supervised_weight: Weight of supervised loss term.
demo_memory_capacity: Int describing capacity of expert demonstration memory.
demo_sampling_ratio: Runtime sampling ratio of expert data.
"""
if network_spec is None:
raise TensorForceError("No network_spec provided.")
if optimizer is None:
self.optimizer = dict(type='adam', learning_rate=1e-3)
else:
self.optimizer = optimizer
if memory is None:
memory = dict(type='replay', capacity=100000)
else:
self.memory = memory
self.network_spec = network_spec
self.device = device
self.session_config = session_config
self.scope = scope
self.saver_spec = saver_spec
self.summary_spec = summary_spec
self.distributed_spec = distributed_spec
self.discount = discount
self.variable_noise = variable_noise
self.states_preprocessing_spec = states_preprocessing_spec
self.explorations_spec = explorations_spec
self.reward_preprocessing_spec = reward_preprocessing_spec
self.distributions_spec = distributions_spec
self.entropy_regularization = entropy_regularization
self.target_sync_frequency = target_sync_frequency
self.target_update_weight = target_update_weight
self.huber_loss = huber_loss
# DQFD always uses double dqn, which is a required key for a q-model.
self.double_q_model = True
self.target_sync_frequency = target_sync_frequency
self.demo_memory_capacity = demo_memory_capacity
self.expert_margin = expert_margin
self.supervised_weight = supervised_weight
# The demo_sampling_ratio, called p in paper, controls ratio of expert vs online training samples
# p = n_demo / (n_demo + n_replay) => n_demo = p * n_replay / (1 - p)
self.demo_batch_size = int(demo_sampling_ratio * batch_size /
(1.0 - demo_sampling_ratio))
assert self.demo_batch_size > 0, 'Check DQFD sampling parameters to ensure ' \
'demo_batch_size is positive. (Calculated {} based on current' \
' parameters)'.format(self.demo_batch_size)
# This is the demonstration memory that we will fill with observations before starting
# the main training loop
super(DQFDAgent, self).__init__(states_spec=states_spec,
actions_spec=actions_spec,
batched_observe=batched_observe,
batch_size=batch_size,
memory=memory,
first_update=first_update,
update_frequency=update_frequency,
repeat_update=repeat_update)
self.demo_memory = Replay(self.states_spec, self.actions_spec,
self.demo_memory_capacity)
def initialize_model(self):
return QDemoModel(
states_spec=self.states_spec,
actions_spec=self.actions_spec,
network_spec=self.network_spec,
device=self.device,
session_config=self.session_config,
scope=self.scope,
saver_spec=self.saver_spec,
summary_spec=self.summary_spec,
distributed_spec=self.distributed_spec,
optimizer=self.optimizer,
discount=self.discount,
variable_noise=self.variable_noise,
states_preprocessing_spec=self.states_preprocessing_spec,
explorations_spec=self.explorations_spec,
reward_preprocessing_spec=self.reward_preprocessing_spec,
distributions_spec=self.distributions_spec,
entropy_regularization=self.entropy_regularization,
target_sync_frequency=self.target_sync_frequency,
target_update_weight=self.target_update_weight,
double_q_model=self.double_q_model,
huber_loss=self.huber_loss,
# TEMP: Random sampling fix
random_sampling_fix=True,
expert_margin=self.expert_margin,
supervised_weight=self.supervised_weight)
def observe(self, reward, terminal):
"""
Adds observations, updates via sampling from memories according to update rate.
DQFD samples from the online replay memory and the demo memory with
the fractions controlled by a hyper parameter p called 'expert sampling ratio.
Args:
reward:
terminal:
"""
super(DQFDAgent, self).observe(reward=reward, terminal=terminal)
if self.timestep >= self.first_update and self.timestep % self.update_frequency == 0:
for _ in xrange(self.repeat_update):
batch = self.demo_memory.get_batch(
batch_size=self.demo_batch_size, next_states=True)
self.model.demonstration_update(states={
name: np.stack(
(batch['states'][name], batch['next_states'][name]))
for name in batch['states']
},
internals=batch['internals'],
actions=batch['actions'],
terminal=batch['terminal'],
reward=batch['reward'])
def import_demonstrations(self, demonstrations):
"""
Imports demonstrations, i.e. expert observations. Note that for large numbers of observations,
set_demonstrations is more appropriate, which directly sets memory contents to an array an expects
a different layout.
Args:
demonstrations: List of observation dicts
"""
for observation in demonstrations:
if self.unique_state:
state = dict(state=observation['states'])
else:
state = observation['states']
if self.unique_action:
action = dict(action=observation['actions'])
else:
action = observation['actions']
self.demo_memory.add_observation(
states=state,
internals=observation['internals'],
actions=action,
terminal=observation['terminal'],
reward=observation['reward'])
def set_demonstrations(self, batch):
"""
Set all demonstrations from batch data. Expects a dict wherein each value contains an array
containing all states, actions, rewards, terminals and internals respectively.
Args:
batch:
"""
self.demo_memory.set_memory(states=batch['states'],
internals=batch['internals'],
actions=batch['actions'],
terminal=batch['terminal'],
reward=batch['reward'])
def pretrain(self, steps):
"""
Computes pretrain updates.
Args:
steps: Number of updates to execute.
"""
for _ in xrange(steps):
# Sample from demo memory.
batch = self.demo_memory.get_batch(batch_size=self.batch_size,
next_states=True)
# Update using both double Q-learning and supervised double_q_loss.
self.model.demonstration_update(states={
name: np.stack(
(batch['states'][name], batch['next_states'][name]))
for name in batch['states']
},
internals=batch['internals'],
actions=batch['actions'],
terminal=batch['terminal'],
reward=batch['reward'])
class DQFDAgent(MemoryAgent):
"""
Deep Q-learning from demonstration (DQFD) agent ([Hester et al., 2017](https://arxiv.org/abs/1704.03732)).
This agent uses DQN to pre-train from demonstration data via an additional supervised loss term.
"""
def __init__(
self,
states_spec,
actions_spec,
batched_observe=1000,
scope='dqfd',
# parameters specific to LearningAgents
summary_spec=None,
network_spec=None,
device=None,
session_config=None,
saver_spec=None,
distributed_spec=None,
optimizer=None,
discount=0.99,
variable_noise=None,
states_preprocessing_spec=None,
explorations_spec=None,
reward_preprocessing_spec=None,
distributions_spec=None,
entropy_regularization=None,
# parameters specific to MemoryAgents
batch_size=32,
memory=None,
first_update=10000,
update_frequency=4,
repeat_update=1,
# parameters specific to DQFD agents
target_sync_frequency=10000,
target_update_weight=1.0,
huber_loss=None,
expert_margin=0.5,
supervised_weight=0.1,
demo_memory_capacity=10000,
demo_sampling_ratio=0.2):
"""
Deep Q-learning from demonstration (DQFD) agent ([Hester et al., 2017](https://arxiv.org/abs/1704.03732)).
This agent uses DQN to pre-train from demonstration data in combination with a supervised loss.
Args:
target_sync_frequency: Interval between optimization calls synchronizing the target network.
target_update_weight: Update weight, 1.0 meaning a full assignment to target network from training network.
huber_loss: Optional flat specifying Huber-loss clipping.
expert_margin: Positive float specifying enforced supervised margin between expert action Q-value and other
Q-values.
supervised_weight: Weight of supervised loss term.
demo_memory_capacity: Int describing capacity of expert demonstration memory.
demo_sampling_ratio: Runtime sampling ratio of expert data.
"""
self.target_sync_frequency = target_sync_frequency
self.target_update_weight = target_update_weight
self.huber_loss = huber_loss
self.expert_margin = expert_margin
self.supervised_weight = supervised_weight
super(DQFDAgent, self).__init__(
states_spec=states_spec,
actions_spec=actions_spec,
batched_observe=batched_observe,
scope=scope,
# parameters specific to LearningAgent
summary_spec=summary_spec,
network_spec=network_spec,
discount=discount,
device=device,
session_config=session_config,
saver_spec=saver_spec,
distributed_spec=distributed_spec,
optimizer=optimizer,
variable_noise=variable_noise,
states_preprocessing_spec=states_preprocessing_spec,
explorations_spec=explorations_spec,
reward_preprocessing_spec=reward_preprocessing_spec,
distributions_spec=distributions_spec,
entropy_regularization=entropy_regularization,
# parameters specific to MemoryAgents
batch_size=batch_size,
memory=memory,
first_update=first_update,
update_frequency=update_frequency,
repeat_update=repeat_update)
# The demo_sampling_ratio, called p in paper, controls ratio of expert vs online training samples
# p = n_demo / (n_demo + n_replay) => n_demo = p * n_replay / (1 - p)
self.demo_memory_capacity = demo_memory_capacity
self.demo_batch_size = int(demo_sampling_ratio * batch_size /
(1.0 - demo_sampling_ratio))
assert self.demo_batch_size > 0, 'Check DQFD sampling parameters to ensure ' \
'demo_batch_size is positive. (Calculated {} based on current' \
' parameters)'.format(self.demo_batch_size)
# This is the demonstration memory that we will fill with observations before starting
# the main training loop
self.demo_memory = Replay(self.states_spec, self.actions_spec,
self.demo_memory_capacity)
def initialize_model(self):
return QDemoModel(
states_spec=self.states_spec,
actions_spec=self.actions_spec,
network_spec=self.network_spec,
device=self.device,
session_config=self.session_config,
scope=self.scope,
saver_spec=self.saver_spec,
summary_spec=self.summary_spec,
distributed_spec=self.distributed_spec,
optimizer=self.optimizer,
discount=self.discount,
variable_noise=self.variable_noise,
states_preprocessing_spec=self.states_preprocessing_spec,
explorations_spec=self.explorations_spec,
reward_preprocessing_spec=self.reward_preprocessing_spec,
distributions_spec=self.distributions_spec,
entropy_regularization=self.entropy_regularization,
target_sync_frequency=self.target_sync_frequency,
target_update_weight=self.target_update_weight,
# DQFD always uses double dqn, which is a required key for a q-model.
double_q_model=True,
huber_loss=self.huber_loss,
# TEMP: Random sampling fix
random_sampling_fix=True,
expert_margin=self.expert_margin,
supervised_weight=self.supervised_weight)
def observe(self, reward, terminal):
"""
Adds observations, updates via sampling from memories according to update rate.
DQFD samples from the online replay memory and the demo memory with
the fractions controlled by a hyper parameter p called 'expert sampling ratio.
"""
super(DQFDAgent, self).observe(reward=reward, terminal=terminal)
if self.timestep >= self.first_update and self.timestep % self.update_frequency == 0:
for _ in xrange(self.repeat_update):
batch = self.demo_memory.get_batch(
batch_size=self.demo_batch_size, next_states=True)
self.model.demonstration_update(states={
name: np.stack(
(batch['states'][name], batch['next_states'][name]))
for name in batch['states']
},
internals=batch['internals'],
actions=batch['actions'],
terminal=batch['terminal'],
reward=batch['reward'])
def import_demonstrations(self, demonstrations):
"""
Imports demonstrations, i.e. expert observations. Note that for large numbers of observations,
set_demonstrations is more appropriate, which directly sets memory contents to an array an expects
a different layout.
Args:
demonstrations: List of observation dicts
"""
for observation in demonstrations:
if self.unique_state:
state = dict(state=observation['states'])
else:
state = observation['states']
if self.unique_action:
action = dict(action=observation['actions'])
else:
action = observation['actions']
self.demo_memory.add_observation(
states=state,
internals=observation['internals'],
actions=action,
terminal=observation['terminal'],
reward=observation['reward'])
def set_demonstrations(self, batch):
"""
Set all demonstrations from batch data. Expects a dict wherein each value contains an array
containing all states, actions, rewards, terminals and internals respectively.
Args:
batch:
"""
self.demo_memory.set_memory(states=batch['states'],
internals=batch['internals'],
actions=batch['actions'],
terminal=batch['terminal'],
reward=batch['reward'])
def pretrain(self, steps):
"""
Computes pre-train updates.
Args:
steps: Number of updates to execute.
"""
for _ in xrange(steps):
# Sample from demo memory.
batch = self.demo_memory.get_batch(batch_size=self.batch_size,
next_states=True)
# Update using both double Q-learning and supervised double_q_loss.
self.model.demonstration_update(states={
name: np.stack(
(batch['states'][name], batch['next_states'][name]))
for name in batch['states']
},
internals=batch['internals'],
actions=batch['actions'],
terminal=batch['terminal'],
reward=batch['reward'])
