def __init__(
# Required
self, states, actions, update, objective, reward_estimation,
# Environment
max_episode_timesteps=None,
# Agent
policy='default', memory=None, optimizer='adam',
# Baseline
baseline_policy=None, baseline_optimizer=None, baseline_objective=None,
# Preprocessing
preprocessing=None,
# Exploration
exploration=0.0, variable_noise=0.0,
# Regularization
l2_regularization=0.0, entropy_regularization=0.0,
# TensorFlow etc
name='agent', device=None, parallel_interactions=1, buffer_observe=True, seed=None,
execution=None, saver=None, summarizer=None, recorder=None, config=None
):
if not hasattr(self, 'spec'):
self.spec = OrderedDict(
agent='tensorforce',
states=states, actions=actions, max_episode_timesteps=max_episode_timesteps,
policy=policy, memory=memory, update=update, optimizer=optimizer,
objective=objective, reward_estimation=reward_estimation,
baseline_policy=baseline_policy, baseline_optimizer=baseline_optimizer,
baseline_objective=baseline_objective,
preprocessing=preprocessing,
exploration=exploration, variable_noise=variable_noise,
l2_regularization=l2_regularization, entropy_regularization=entropy_regularization,
name=name, device=device, parallel_interactions=parallel_interactions,
buffer_observe=buffer_observe, seed=seed, execution=execution, saver=saver,
summarizer=summarizer, recorder=recorder, config=config
)
if isinstance(update, int) or update['unit'] == 'timesteps':
if parallel_interactions > 1:
raise TensorforceError.value(
name='agent', argument='update', value=update,
condition='parallel_interactions > 1'
)
if buffer_observe is not True:
raise TensorforceError.invalid(
name='agent', argument='buffer_observe', condition='update[unit] = timesteps'
)
buffer_observe = False
if buffer_observe is True and parallel_interactions == 1 and summarizer is not None:
buffer_observe = False
super().__init__(
states=states, actions=actions, max_episode_timesteps=max_episode_timesteps,
parallel_interactions=parallel_interactions, buffer_observe=buffer_observe, seed=seed,
recorder=recorder
)
if isinstance(update, int):
update = dict(unit='timesteps', batch_size=update)
reward_estimation = dict(reward_estimation)
if reward_estimation['horizon'] == 'episode':
if max_episode_timesteps is None:
raise TensorforceError.value(
name='agent', argument='reward_estimation[horizon]', value='episode',
condition='max_episode_timesteps is None'
)
reward_estimation['horizon'] = max_episode_timesteps
# TEMPORARY TODO: State value doesn't exist for Beta
if isinstance(baseline_policy, dict) and \
baseline_policy.get('type') in (None, 'default', 'parametrized_distributions') and \
'distributions' not in baseline_policy:
baseline_policy['distributions'] = dict(float='gaussian')
if (reward_estimation.get('estimate_horizon') is not False or \
reward_estimation.get('estimate_advantage') is not False or \
baseline_policy is not None or baseline_optimizer is not None or \
baseline_objective is not None) and isinstance(policy, dict) and \
policy.get('type') in (None, 'default', 'parametrized_distributions') and \
'distributions' not in policy:
policy['distributions'] = dict(float='gaussian')
self.model = TensorforceModel(
# Model
name=name, device=device, parallel_interactions=self.parallel_interactions,
buffer_observe=self.buffer_observe, seed=seed, execution=execution, saver=saver,
summarizer=summarizer, config=config, states=self.states_spec,
actions=self.actions_spec, preprocessing=preprocessing, exploration=exploration,
variable_noise=variable_noise, l2_regularization=l2_regularization,
# TensorforceModel
policy=policy, memory=memory, update=update, optimizer=optimizer, objective=objective,
reward_estimation=reward_estimation, baseline_policy=baseline_policy,
baseline_optimizer=baseline_optimizer, baseline_objective=baseline_objective,
entropy_regularization=entropy_regularization,
max_episode_timesteps=max_episode_timesteps
)
self.experience_size = self.model.estimator.capacity
def __init__(
# Required
self,
states,
actions,
update,
optimizer,
objective,
reward_estimation,
# Environment
max_episode_timesteps=None,
# Agent
policy='auto',
memory=None,
# Baseline
baseline=None,
baseline_optimizer=None,
baseline_objective=None,
# Regularization
l2_regularization=0.0,
entropy_regularization=0.0,
# Preprocessing
state_preprocessing='linear_normalization',
reward_preprocessing=None,
# Exploration
exploration=0.0,
variable_noise=0.0,
# Parallel interactions
parallel_interactions=1,
# Config, saver, summarizer, tracking, recorder
config=None,
saver=None,
summarizer=None,
tracking=None,
recorder=None,
# Deprecated
baseline_policy=None,
name=None,
buffer_observe=None,
device=None,
seed=None):
if 'estimate_actions' in reward_estimation:
raise TensorforceError.deprecated(
name='Agent',
argument='reward_estimation[estimate_actions]',
replacement='reward_estimation[estimate_action_values]')
if 'estimate_terminal' in reward_estimation:
raise TensorforceError.deprecated(
name='Agent',
argument='reward_estimation[estimate_terminal]',
replacement='reward_estimation[estimate_terminals]')
if summarizer is not None and 'labels' in summarizer:
raise TensorforceError.deprecated(
name='Agent',
argument='summarizer[labels]',
replacement='summarizer[summaries]')
if baseline_policy is not None:
raise TensorforceError.deprecated(name='Agent',
argument='baseline_policy',
replacement='baseline')
if name is not None:
raise TensorforceError.deprecated(name='Agent',
argument='name',
replacement='config[name]')
if buffer_observe is not None:
raise TensorforceError.deprecated(
name='Agent',
argument='buffer_observe',
replacement='config[buffer_observe]')
if device is not None:
raise TensorforceError.deprecated(name='Agent',
argument='device',
replacement='config[device]')
if seed is not None:
raise TensorforceError.deprecated(name='Agent',
argument='seed',
replacement='config[seed]')
if not hasattr(self, 'spec'):
self.spec = OrderedDict(
agent='tensorforce',
# Environment
states=states,
actions=actions,
max_episode_timesteps=max_episode_timesteps,
# Agent
policy=policy,
memory=memory,
update=update,
optimizer=optimizer,
objective=objective,
reward_estimation=reward_estimation,
# Baseline
baseline=baseline,
baseline_optimizer=baseline_optimizer,
baseline_objective=baseline_objective,
# Regularization
l2_regularization=l2_regularization,
entropy_regularization=entropy_regularization,
# Preprocessing
state_preprocessing=state_preprocessing,
reward_preprocessing=reward_preprocessing,
# Exploration
exploration=exploration,
variable_noise=variable_noise,
# Parallel interactions
parallel_interactions=parallel_interactions,
# Config, saver, summarizer, recorder
config=config,
saver=saver,
summarizer=summarizer,
tracking=tracking,
recorder=recorder)
if memory is None:
memory = dict(type='recent')
if isinstance(update, int):
update = dict(unit='timesteps', batch_size=update)
if config is None:
config = dict()
else:
config = dict(config)
# TODO: should this change if summarizer is specified?
if parallel_interactions > 1:
if 'buffer_observe' not in config:
if max_episode_timesteps is None:
raise TensorforceError.required(
name='Agent',
argument='max_episode_timesteps',
condition='parallel_interactions > 1')
config['buffer_observe'] = 'episode'
# elif config['buffer_observe'] < max_episode_timesteps:
# raise TensorforceError.value(
# name='Agent', argument='config[buffer_observe]',
# hint='< max_episode_timesteps', condition='parallel_interactions > 1'
# )
elif update['unit'] == 'timesteps':
update_frequency = update.get('frequency', update['batch_size'])
if 'buffer_observe' not in config:
if isinstance(update_frequency, int):
config['buffer_observe'] = update_frequency
else:
config['buffer_observe'] = 1
elif isinstance(
update_frequency,
int) and (config['buffer_observe'] == 'episode'
or config['buffer_observe'] > update_frequency):
raise TensorforceError.value(
name='Agent',
argument='config[buffer_observe]',
value=config['buffer_observe'],
hint='> update[frequency]',
condition='update[unit] = "timesteps"')
elif update['unit'] == 'episodes':
if 'buffer_observe' not in config:
config['buffer_observe'] = 'episode'
# reward_estimation = dict(reward_estimation)
# if reward_estimation['horizon'] == 'episode':
# if max_episode_timesteps is None:
# raise TensorforceError.required(
# name='Agent', argument='max_episode_timesteps',
# condition='reward_estimation[horizon] = "episode"'
# )
# reward_estimation['horizon'] = max_episode_timesteps
super().__init__(states=states,
actions=actions,
max_episode_timesteps=max_episode_timesteps,
parallel_interactions=parallel_interactions,
config=config,
recorder=recorder)
self.model = TensorforceModel(
states=self.states_spec,
actions=self.actions_spec,
max_episode_timesteps=self.max_episode_timesteps,
policy=policy,
memory=memory,
update=update,
optimizer=optimizer,
objective=objective,
reward_estimation=reward_estimation,
baseline=baseline,
baseline_optimizer=baseline_optimizer,
baseline_objective=baseline_objective,
l2_regularization=l2_regularization,
entropy_regularization=entropy_regularization,
state_preprocessing=state_preprocessing,
reward_preprocessing=reward_preprocessing,
exploration=exploration,
variable_noise=variable_noise,
parallel_interactions=self.parallel_interactions,
config=self.config,
saver=saver,
summarizer=summarizer,
tracking=tracking)
def __init__(
self,
# --- required ---
# Environment
states,
actions,
# Agent
update,
objective,
reward_estimation,
# --- default ---
# Environment
max_episode_timesteps=None,
# Agent
policy=None,
network='auto',
memory=None,
optimizer='adam',
# Baseline
baseline_policy=None,
baseline_network=None,
baseline_optimizer=None,
baseline_objective=None,
# Preprocessing
preprocessing=None,
# Exploration
exploration=0.0,
variable_noise=0.0,
# Regularization
l2_regularization=0.0,
entropy_regularization=0.0,
# TensorFlow etc
name='agent',
device=None,
parallel_interactions=1,
buffer_observe=True,
seed=None,
execution=None,
saver=None,
summarizer=None,
recorder=None,
config=None):
if not hasattr(self, 'spec'):
self.spec = OrderedDict(
agent='tensorforce',
states=states,
actions=actions,
max_episode_timesteps=max_episode_timesteps,
policy=policy,
network=network,
memory=memory,
update=update,
optimizer=optimizer,
objective=objective,
reward_estimation=reward_estimation,
baseline_policy=baseline_policy,
baseline_network=baseline_network,
baseline_optimizer=baseline_optimizer,
baseline_objective=baseline_objective,
preprocessing=preprocessing,
exploration=exploration,
variable_noise=variable_noise,
l2_regularization=l2_regularization,
entropy_regularization=entropy_regularization,
name=name,
device=device,
parallel_interactions=parallel_interactions,
seed=seed,
execution=execution,
saver=saver,
summarizer=summarizer,
recorder=recorder,
config=config)
if buffer_observe is True and parallel_interactions == 1 and summarizer is not None:
buffer_observe = False
super().__init__(states=states,
actions=actions,
max_episode_timesteps=max_episode_timesteps,
parallel_interactions=parallel_interactions,
buffer_observe=buffer_observe,
seed=seed,
recorder=recorder)
if isinstance(update, int):
update = dict(unit='timesteps', batch_size=update)
if reward_estimation['horizon'] == 'episode':
if max_episode_timesteps is None:
raise TensorforceError.unexpected()
reward_estimation['horizon'] = max_episode_timesteps
if memory is None:
# predecessor/successor?
if max_episode_timesteps is None or not isinstance(update['batch_size'], int) \
or not isinstance(reward_estimation['horizon'], int):
raise TensorforceError.unexpected()
if update['unit'] == 'timesteps':
memory = update['batch_size'] + max_episode_timesteps + \
reward_estimation['horizon']
# memory = ceil(update['batch_size'] / max_episode_timesteps) * max_episode_timesteps
# memory += int(update['batch_size'] / max_episode_timesteps >= 1.0)
elif update['unit'] == 'episodes':
memory = update['batch_size'] * max_episode_timesteps + \
max(reward_estimation['horizon'], max_episode_timesteps)
memory = max(memory, min(self.buffer_observe,
max_episode_timesteps))
self.model = TensorforceModel(
# Model
name=name,
device=device,
parallel_interactions=self.parallel_interactions,
buffer_observe=self.buffer_observe,
execution=execution,
saver=saver,
summarizer=summarizer,
config=config,
states=self.states_spec,
actions=self.actions_spec,
preprocessing=preprocessing,
exploration=exploration,
variable_noise=variable_noise,
l2_regularization=l2_regularization,
# TensorforceModel
policy=policy,
network=network,
memory=memory,
update=update,
optimizer=optimizer,
objective=objective,
reward_estimation=reward_estimation,
baseline_policy=baseline_policy,
baseline_network=baseline_network,
baseline_optimizer=baseline_optimizer,
baseline_objective=baseline_objective,
entropy_regularization=entropy_regularization)
assert max_episode_timesteps is None or self.model.memory.capacity > max_episode_timesteps
def __init__(
self,
# --- required ---
# Environment
states,
actions,
# Agent
update,
objective,
reward_estimation,
# --- default ---
# Environment
max_episode_timesteps=None,
# Agent
policy='default',
memory=None,
optimizer='adam',
# Baseline
baseline_policy=None,
baseline_optimizer=None,
baseline_objective=None,
# Preprocessing
preprocessing=None,
# Exploration
exploration=0.0,
variable_noise=0.0,
# Regularization
l2_regularization=0.0,
entropy_regularization=0.0,
# TensorFlow etc
name='agent',
device=None,
parallel_interactions=1,
buffer_observe=True,
seed=None,
execution=None,
saver=None,
summarizer=None,
recorder=None,
config=None):
if not hasattr(self, 'spec'):
self.spec = OrderedDict(
agent='tensorforce',
states=states,
actions=actions,
max_episode_timesteps=max_episode_timesteps,
policy=policy,
memory=memory,
update=update,
optimizer=optimizer,
objective=objective,
reward_estimation=reward_estimation,
baseline_policy=baseline_policy,
baseline_optimizer=baseline_optimizer,
baseline_objective=baseline_objective,
preprocessing=preprocessing,
exploration=exploration,
variable_noise=variable_noise,
l2_regularization=l2_regularization,
entropy_regularization=entropy_regularization,
name=name,
device=device,
parallel_interactions=parallel_interactions,
buffer_observe=buffer_observe,
seed=seed,
execution=execution,
saver=saver,
summarizer=summarizer,
recorder=recorder,
config=config)
if isinstance(update, int) or update['unit'] == 'timesteps':
if parallel_interactions > 1:
raise TensorforceError.value(
name='agent',
argument='update',
value=update,
condition='parallel_interactions > 1')
if buffer_observe is not True:
raise TensorforceError.invalid(
name='agent',
argument='buffer_observe',
condition='update[unit] = timesteps')
buffer_observe = False
if buffer_observe is True and parallel_interactions == 1 and summarizer is not None:
buffer_observe = False
super().__init__(states=states,
actions=actions,
max_episode_timesteps=max_episode_timesteps,
parallel_interactions=parallel_interactions,
buffer_observe=buffer_observe,
seed=seed,
recorder=recorder)
if isinstance(update, int):
update = dict(unit='timesteps', batch_size=update)
reward_estimation = dict(reward_estimation)
if reward_estimation['horizon'] == 'episode':
if max_episode_timesteps is None:
raise TensorforceError.value(
name='agent',
argument='reward_estimation[horizon]',
value='episode',
condition='max_episode_timesteps is None')
reward_estimation['horizon'] = max_episode_timesteps
if 'capacity' not in reward_estimation:
# TODO: Doesn't take network horizon into account, needs to be set internally to max
# if isinstance(reward_estimation['horizon'], int):
# reward_estimation['capacity'] = max(
# self.buffer_observe, reward_estimation['horizon'] + 2
# )
if max_episode_timesteps is None:
raise TensorforceError.required(
name='agent',
argument='reward_estimation[capacity]',
condition='max_episode_timesteps is None')
if isinstance(reward_estimation['horizon'], int):
reward_estimation['capacity'] = max(
max_episode_timesteps, reward_estimation['horizon'])
else:
reward_estimation['capacity'] = max_episode_timesteps
self.experience_size = reward_estimation['capacity']
if memory is None or (isinstance(memory, dict)
and 'capacity' not in memory):
# predecessor/successor?
if max_episode_timesteps is None:
raise TensorforceError.required(
name='agent',
argument='memory',
condition='max_episode_timesteps is None')
elif not isinstance(update['batch_size'], int):
raise TensorforceError.required(
name='agent',
argument='memory',
condition='update[batch_size] not int')
elif not isinstance(reward_estimation['horizon'], int):
raise TensorforceError.required(
name='agent',
argument='memory',
condition='reward_estimation[horizon] not int')
if update['unit'] == 'timesteps':
capacity = update['batch_size'] + max_episode_timesteps + \
reward_estimation['horizon']
# capacity = ceil(update['batch_size'] / max_episode_timesteps) * max_episode_timesteps
# capacity += int(update['batch_size'] / max_episode_timesteps >= 1.0)
elif update['unit'] == 'episodes':
capacity = update['batch_size'] * max_episode_timesteps + \
max(reward_estimation['horizon'], max_episode_timesteps)
capacity = max(capacity,
min(self.buffer_observe, max_episode_timesteps))
if memory is None:
memory = capacity
else:
memory['capacity'] = capacity
self.model = TensorforceModel(
# Model
name=name,
device=device,
parallel_interactions=self.parallel_interactions,
buffer_observe=self.buffer_observe,
seed=seed,
execution=execution,
saver=saver,
summarizer=summarizer,
config=config,
states=self.states_spec,
actions=self.actions_spec,
preprocessing=preprocessing,
exploration=exploration,
variable_noise=variable_noise,
l2_regularization=l2_regularization,
# TensorforceModel
policy=policy,
memory=memory,
update=update,
optimizer=optimizer,
objective=objective,
reward_estimation=reward_estimation,
baseline_policy=baseline_policy,
baseline_optimizer=baseline_optimizer,
baseline_objective=baseline_objective,
entropy_regularization=entropy_regularization)
if max_episode_timesteps is not None and \
self.model.memory.capacity <= max_episode_timesteps:
raise TensorforceError.value(name='agent',
argument='memory.capacity',
value=self.model.memory.capacity,
hint='<= max_episode_timesteps')