def __init__(self,
name,
l2_regularization=None,
summary_labels=None,
device=None):
# name
if not util.is_valid_name(name=name):
raise TensorforceError.value(name='module',
argument='name',
value=name)
# l2_regularization
if l2_regularization is not None and not isinstance(
l2_regularization, float):
raise TensorforceError.type(name='module',
argument='l2_regularization',
value=l2_regularization)
if l2_regularization is not None and l2_regularization < 0.0:
raise TensorforceError.value(name='module',
argument='l2_regularization',
value=l2_regularization)
# summary_labels
if summary_labels is not None and \
not all(isinstance(label, str) for label in summary_labels):
raise TensorforceError.type(name='module',
argument='summary_labels',
value=summary_labels)
# device
# ???
# Attributes specified via constructor arguments
self.name = name
self.l2_regularization = l2_regularization
self.summary_labels = None if summary_labels is None else set(
summary_labels)
self.device = device
# Otherwise inherit arguments
if self.l2_regularization is None:
self.l2_regularization = Module.inherit_l2_regularization
if self.summary_labels is None:
self.summary_labels = Module.inherit_summary_labels
# Internal attributes
self.parent = None
self.scope = None
self.is_subscope = None
self.modules = OrderedDict()
self.trainable_modules = OrderedDict()
self.is_initialized = False
self.variables = None
self.trainable_variables = None
self.regularized_variables = None
if not Module.is_add_module:
Module.global_scope = list()
Module.global_tensors_spec = OrderedDict()
def __init__(self, name, device=None, summary_labels=None, l2_regularization=None):
# Internal attributes
self.parent = Module.set_parent
# self.scope = None
self.is_subscope = None
self.modules = OrderedDict()
self.trainable_modules = OrderedDict()
self.saved_modules = OrderedDict()
self.is_initialized = False
self.variables = None
self.trainable_variables = None
self.saved_variables = None
self.output_tensors = None
self.query_tensors = None
self.available_summaries = None
# name
if not util.is_valid_name(name=name):
raise TensorforceError.value(name='module', argument='name', value=name)
# summary_labels
if summary_labels is not None and \
not all(isinstance(label, str) for label in summary_labels):
raise TensorforceError.value(
name='module', argument='summary_labels', value=summary_labels
)
# device
# ???
self.name = name
self.device = device
if summary_labels is None:
# Otherwise inherit arguments
self.summary_labels = Module.inherit_summary_labels
elif summary_labels == 'all':
self.summary_labels = summary_labels
else:
self.summary_labels = set(summary_labels)
if not Module.is_add_module:
Module.global_scope = list()
Module.global_tensors_spec = OrderedDict()
if Module.inherit_l2_regularization is None and l2_regularization is None:
self.l2_regularization = None # otherwise infinite recursion
elif l2_regularization is not None:
from tensorforce.core import parameter_modules
self.l2_regularization = None # for first module
self.l2_regularization = self.add_module(
name='l2-regularization', module=l2_regularization, modules=parameter_modules,
is_trainable=False, dtype='float'
)
else:
# Otherwise inherit arguments
self.l2_regularization = Module.inherit_l2_regularization
def add_placeholder(self, name, dtype, shape, batched, default=None):
# name
name = name + '-input'
if not util.is_valid_name(name=name):
raise TensorforceError.value(name='placeholder',
argument='name',
value=name)
# dtype
if not util.is_valid_type(dtype=dtype):
raise TensorforceError.value(name='placeholder',
argument='dtype',
value=dtype)
# shape
if not util.is_iterable(x=shape) or \
not all(isinstance(num_dims, int) for num_dims in shape):
raise TensorforceError.type(name='placeholder',
argument='shape',
value=shape)
elif not all(num_dims > 0 for num_dims in shape):
raise TensorforceError.value(name='placeholder',
argument='shape',
value=shape)
# batched
if not isinstance(batched, bool):
raise TensorforceError.type(name='placeholder',
argument='batched',
value=batched)
# default
if default is not None:
if batched:
raise TensorforceError.unexpected()
elif not isinstance(default, tf.Tensor):
raise TensorforceError.unexpected()
elif util.dtype(x=default) != dtype:
raise TensorforceError.unexpected()
# Placeholder
if batched:
shape = (None, ) + shape
if default is None:
dtype = util.tf_dtype(dtype=dtype)
placeholder = tf.placeholder(dtype=dtype, shape=shape, name=name)
else:
# check dtype and shape !!!
placeholder = tf.placeholder_with_default(input=default,
shape=shape,
name=name)
return placeholder
def get_module_class_and_kwargs(
name, module=None, modules=None, default_module=None, disable_first_arg=False, **kwargs
):
# name
if not util.is_valid_name(name=name):
raise TensorforceError.value(name='Module.add_module', argument='name', value=name)
# module
# ???
# modules
if modules is not None and not isinstance(modules, dict):
raise TensorforceError.type(
name='Module.add_module', argument='modules', dtype=type(modules)
)
# default_module
# ???
if isinstance(module, dict):
# Dictionary module specification (type either given via 'type' or 'default_module')
util.deep_disjoint_update(target=kwargs, source=module)
module = kwargs.pop('type', default_module)
return Module.get_module_class_and_kwargs(
name=name, module=module, modules=modules, default_module=default_module,
disable_first_arg=True, **kwargs
)
elif isinstance(module, str):
if os.path.isfile(module):
# JSON file module specification
with open(module, 'r') as fp:
module = json.load(fp=fp)
return Module.get_module_class_and_kwargs(
name=name, module=module, modules=modules, default_module=default_module,
disable_first_arg=True, **kwargs
)
elif '.' in module:
# Library module specification
library_name, module_name = module.rsplit('.', 1)
library = importlib.import_module(name=library_name)
module = getattr(library, module_name)
return Module.get_module_class_and_kwargs(
name=name, module=module, modules=modules, default_module=default_module,
disable_first_arg=True, **kwargs
)
elif modules is not None and module in modules:
# Keyword module specification
return Module.get_module_class_and_kwargs(
name=name, module=modules[module], default_module=default_module,
disable_first_arg=True, **kwargs
)
elif 'default' in modules or default_module is not None:
# Default module specification
if '_first_arg' in kwargs:
raise TensorforceError.invalid(name='Module.add_module', argument='_first_arg')
if module is not None:
if disable_first_arg:
raise TensorforceError.value(
name='Module.add_module', argument='module', value=module
)
kwargs['_first_arg'] = module
if default_module is None:
default_module = modules['default']
return Module.get_module_class_and_kwargs(
name=name, module=default_module, modules=modules, **kwargs
)
else:
raise TensorforceError.value(
name='Module.add_module', argument='module', value=module
)
elif not callable(module) and ('default' in modules or default_module is not None):
# Default module specification
if '_first_arg' in kwargs:
raise TensorforceError.invalid(name='Module.add_module', argument='_first_arg')
if module is not None:
kwargs['_first_arg'] = module
if default_module is None:
default_module = modules['default']
return Module.get_module_class_and_kwargs(
name=name, module=default_module, modules=modules, **kwargs
)
elif callable(module):
# for key, arg in kwargs.items():
# assert arg is not None, (key, arg)
# if arg is None:
# assert False
# kwargs.pop(key)
first_arg = kwargs.pop('_first_arg', None)
return module, first_arg, kwargs
else:
raise TensorforceError.value(name='Module.add_module', argument='module', value=module)
def add_variable(
self, name, dtype, shape, is_trainable, initializer='zeros', is_saved=True, summarize=None,
shared=None
):
# name
if not util.is_valid_name(name=name):
raise TensorforceError.value(name='Module.add_variable', argument='name', value=name)
elif name in self.variables:
raise TensorforceError.exists(name='variable', value=name)
# dtype
if not util.is_valid_type(dtype=dtype):
raise TensorforceError.value(name='Module.add_variable', argument='dtype', value=dtype)
# shape
if not util.is_iterable(x=shape) or not all(isinstance(dims, int) for dims in shape):
raise TensorforceError.value(name='Module.add_variable', argument='shape', value=shape)
elif not all(dims > 0 for dims in shape):
raise TensorforceError.value(name='Module.add_variable', argument='shape', value=shape)
# is_trainable
if not isinstance(is_trainable, bool):
raise TensorforceError.type(
name='Module.add_variable', argument='is_trainable', dtype=type(is_trainable)
)
elif is_trainable and dtype != 'float':
raise TensorforceError.value(
name='Module.add_variable', argument='is_trainable', value=is_trainable,
condition='dtype != float'
)
# initializer
initializer_names = (
'normal', 'normal-relu', 'orthogonal', 'orthogonal-relu', 'zeros', 'ones'
)
if not isinstance(initializer, (util.py_dtype(dtype=dtype), np.ndarray, tf.Tensor)) and \
initializer not in initializer_names:
raise TensorforceError.value(
name='Module.add_variable', argument='initializer', value=initializer
)
elif isinstance(initializer, np.ndarray) and \
initializer.dtype != util.np_dtype(dtype=dtype):
raise TensorforceError.type(
name='Module.add_variable', argument='initializer', dtype=type(initializer)
)
elif isinstance(initializer, tf.Tensor) and util.dtype(x=initializer) != dtype:
raise TensorforceError.type(
name='Module.add_variable', argument='initializer', dtype=type(initializer)
)
# is_saved
if not isinstance(is_saved, bool):
raise TensorforceError.type(
name='Module.add_variable', argument='is_saved', dtype=type(is_saved)
)
# summarize
if summarize is not None and not isinstance(summarize, bool):
raise TensorforceError.type(
name='Module.add_variable', argument='summarize', dtype=type(summarize)
)
# shared
if shared is not None and not isinstance(shared, str):
raise TensorforceError.type(
name='Module.add_variable', argument='shared',dtype=type(shared)
)
variable = None
if shared is not None and len(self.graph.get_collection(name=shared)) > 0:
# Retrieve shared variable from TensorFlow
collection = self.graph.get_collection(name=shared)
if len(collection) > 1:
raise TensorforceError.unexpected()
variable = collection[0]
else:
tf_dtype = util.tf_dtype(dtype=dtype)
# Variable initializer
if isinstance(initializer, util.py_dtype(dtype=dtype)):
initializer = tf.constant(value=initializer, dtype=tf_dtype, shape=shape)
elif isinstance(initializer, np.ndarray):
if initializer.shape != shape:
raise TensorforceError.mismatch(
name='Module.add_variable', value1='shape', value2='initializer'
)
initializer = tf.constant(value=initializer, dtype=tf_dtype)
elif isinstance(initializer, tf.Tensor):
if util.shape(x=initializer) != shape:
raise TensorforceError.mismatch(
name='Module.add_variable', value1='shape', value2='initializer'
)
initializer = initializer
elif not isinstance(initializer, str):
raise TensorforceError("Invalid variable initializer: {}".format(initializer))
elif initializer[:6] == 'normal':
if dtype != 'float':
raise TensorforceError(
message="Invalid variable initializer value for non-float variable: {}.".format(
initializer
)
)
if initializer[6:] == '-relu':
stddev = min(0.1, sqrt(2.0 / util.product(xs=shape[:-1])))
else:
stddev = min(0.1, sqrt(2.0 / (util.product(xs=shape[:-1]) + shape[-1])))
initializer = tf.random.normal(shape=shape, stddev=stddev, dtype=tf_dtype)
elif initializer[:10] == 'orthogonal':
if dtype != 'float':
raise TensorforceError(
message="Invalid variable initializer value for non-float variable: {}.".format(
initializer
)
)
if len(shape) < 2:
raise TensorforceError(
message="Invalid variable initializer value for 0/1-rank variable: {}.".format(
initializer
)
)
normal = np.random.normal(size=(util.product(xs=shape[:-1]), shape[-1]))
u, _, v = np.linalg.svd(a=normal, full_matrices=False)
orthogonal = u if u.shape[1] == shape[-1] else v
if initializer[10:] == '-relu':
orthogonal = orthogonal * sqrt(2.0)
initializer = tf.constant(value=orthogonal.reshape(shape), dtype=tf_dtype)
elif initializer == 'zeros':
initializer = tf.zeros(shape=shape, dtype=tf_dtype)
elif initializer == 'ones':
initializer = tf.ones(shape=shape, dtype=tf_dtype)
# Variable
variable = tf.Variable(
initial_value=initializer, trainable=is_trainable, validate_shape=True, name=name,
dtype=tf_dtype, shape=shape
)
# Register shared variable with TensorFlow
if shared is not None:
self.graph.add_to_collection(name=shared, value=variable)
# Register variable
self.variables[name] = variable
if is_trainable:
self.trainable_variables[name] = variable
if is_saved:
self.saved_variables[name] = variable
# Add summary
if (summarize is None and is_trainable) or summarize:
variable = self.add_summary(
label='variables', name=name, tensor=variable, mean_variance=True
)
variable = self.add_summary(label='variables-histogram', name=name, tensor=variable)
return variable
def initialize(self):
# Check whether module is already initialized
if self.is_initialized:
raise TensorforceError(message="Module is already initialized.")
# Set internal attributes
self.is_initialized = True
self.variables = OrderedDict()
self.trainable_variables = OrderedDict()
self.saved_variables = OrderedDict()
self.output_tensors = dict()
self.query_tensors = dict()
self.available_summaries = set()
if self.parent is None:
Module.global_scope = list()
Module.while_counter = 0
Module.cond_counter = 0
# Global timestep
self.global_timestep = self.add_variable(
name='global-timestep', dtype='long', shape=(), is_trainable=False,
initializer='zeros', shared='global-timestep'
)
collection = self.graph.get_collection(name='global_step')
if len(collection) == 0:
self.graph.add_to_collection(name='global_step', value=self.global_timestep)
if self.summarizer_spec is not None:
with tf.name_scope(name='summarizer'):
directory = self.summarizer_spec['directory']
if os.path.isdir(directory):
directories = sorted(
d for d in os.listdir(directory)
if os.path.isdir(os.path.join(directory, d))
and d.startswith('summary-')
)
else:
os.makedirs(directory)
directories = list()
max_summaries = self.summarizer_spec.get('max-summaries', 5)
if len(directories) > max_summaries - 1:
for subdir in directories[:len(directories) - max_summaries + 1]:
subdir = os.path.join(directory, subdir)
os.remove(os.path.join(subdir, os.listdir(subdir)[0]))
os.rmdir(subdir)
logdir = os.path.join(directory, time.strftime('summary-%Y%m%d-%H%M%S'))
flush_millis = (self.summarizer_spec.get('flush', 10) * 1000)
self.summarizer = tf.summary.create_file_writer(
logdir=logdir, max_queue=None, flush_millis=flush_millis,
filename_suffix=None
)
self.summarizer_init = self.summarizer.init()
self.summarizer_flush = self.summarizer.flush()
self.summarizer_close = self.summarizer.close()
default_summarizer = self.summarizer.as_default()
default_summarizer.__enter__()
if self.summary_labels == 'all' or 'graph' in self.summary_labels:
pass
# TensorFlow device and scope
Module.global_scope.append(self.name)
if self.device is not None:
self.device = tf.device(device_name_or_function=self.device)
self.device.__enter__()
self.scope = tf.name_scope(name=self.name)
with self.scope:
if self.parent is None:
# with tf.device(device_name_or_function=(self.global_model.device if self.global_model else self.device)):
if self.summarizer_spec is not None:
Module.global_summary_step = 'timestep'
condition = tf.constant(value=True, dtype=util.tf_dtype(dtype='bool'))
record_summaries = tf.summary.record_if(condition=condition)
record_summaries.__enter__()
# Global episode
self.global_episode = self.add_variable(
name='global-episode', dtype='long', shape=(), is_trainable=False,
initializer='zeros', shared='global-episode'
)
# Global update
self.global_update = self.add_variable(
name='global-update', dtype='long', shape=(), is_trainable=False,
initializer='zeros', shared='global-update'
)
Module.global_tensors = OrderedDict(
timestep=self.global_timestep, episode=self.global_episode,
update=self.global_update
)
if self.summarizer_spec is not None:
record_summaries.__exit__(None, None, None)
Module.global_summary_step = 'update'
if 'frequency' not in self.summarizer_spec or \
isinstance(self.summarizer_spec['frequency'], int):
condition = tf.constant(value=True, dtype=util.tf_dtype(dtype='bool'))
elif 'variables' in self.summarizer_spec['frequency']:
step = Module.retrieve_tensor(name=Module.global_summary_step)
frequency = tf.constant(
value=self.summarizer_spec['frequency']['variables'],
dtype=util.tf_dtype(dtype='long')
)
zero = tf.constant(value=0, dtype=util.tf_dtype(dtype='long'))
condition = (
lambda: tf.math.equal(x=tf.math.mod(x=step, y=frequency), y=zero)
)
else:
condition = tf.constant(value=False, dtype=util.tf_dtype(dtype='bool'))
record_summaries = tf.summary.record_if(condition=condition)
record_summaries.__enter__()
for module in self.modules.values():
module.initialize()
self.tf_initialize()
if self.parent is None and self.summarizer_spec is not None:
record_summaries.__exit__(None, None, None)
Module.global_summary_step = None
self.scope = None
if self.device is not None:
self.device.__exit__(None, None, None)
Module.global_scope.pop()
if self.parent is None:
assert len(Module.global_scope) == 0
Module.global_scope = None
Module.while_counter = None
Module.cond_counter = None
Module.global_tensors = None
# Internal TensorFlow functions, prefixed by 'tf_'
for attribute in sorted(dir(self)):
if attribute.startswith('tf_') and attribute != 'tf_initialize':
function_name = attribute[3:]
if not util.is_valid_name(name=function_name):
raise TensorforceError.unexpected()
if hasattr(self, function_name):
raise TensorforceError.unexpected()
tf_function = getattr(self, attribute)
if not callable(tf_function):
raise TensorforceError.unexpected()
function = self.create_tf_function(
name='{}.{}'.format(self.name, function_name), tf_function=tf_function
)
setattr(self, function_name, function)
# API TensorFlow functions, prefixed by 'api_'
for attribute in sorted(dir(self)):
if attribute.startswith('api_'):
function_name = attribute[4:]
assert hasattr(self, 'config')
if self.config is not None and 'api_functions' in self.config and \
function_name not in self.config['api_functions']:
continue
if function_name in ('act', 'independent_act'):
Module.global_summary_step = 'timestep'
elif function_name in ('observe', 'experience'):
Module.global_summary_step = 'episode'
elif function_name == 'update':
Module.global_summary_step = 'update'
if self.summarizer_spec is not None:
if 'frequency' not in self.summarizer_spec:
condition = tf.constant(value=True, dtype=util.tf_dtype(dtype='bool'))
elif isinstance(self.summarizer_spec['frequency'], int):
if function_name in ('act', 'independent_act'):
step = self.global_timestep
frequency = tf.constant(
value=self.summarizer_spec['frequency'],
dtype=util.tf_dtype(dtype='long')
)
zero = tf.constant(value=0, dtype=util.tf_dtype(dtype='long'))
condition = (
lambda: tf.math.equal(x=tf.math.mod(x=step, y=frequency), y=zero)
)
elif function_name in ('reset', 'independent_act'):
condition = tf.constant(value=False, dtype=util.tf_dtype(dtype='bool'))
else:
condition = tf.constant(value=True, dtype=util.tf_dtype(dtype='bool'))
elif function_name in self.summarizer_spec['frequency']:
if function_name in ('act', 'independent_act'):
step = self.global_timestep
elif function_name in ('observe', 'experience'):
step = self.global_episode
elif function_name == 'update':
step = self.global_update
elif function_name == 'reset':
raise TensorforceError.value(
name='module', argument='summarizer[frequency]',
value=function_name,
hint='not in {act,experience,observe,update}'
)
else:
raise TensorforceError.value(
name='module', argument='summarizer[frequency]',
value=function_name,
hint='not in {act,experience,observe,update}'
)
frequency = tf.constant(
value=self.summarizer_spec['frequency'][function_name],
dtype=util.tf_dtype(dtype='long')
)
zero = tf.constant(value=0, dtype=util.tf_dtype(dtype='long'))
condition = (
lambda: tf.math.equal(x=tf.math.mod(x=step, y=frequency), y=zero)
)
else:
condition = tf.constant(value=False, dtype=util.tf_dtype(dtype='bool'))
record_summaries = tf.summary.record_if(condition=condition)
record_summaries.__enter__()
if not util.is_valid_name(name=function_name):
raise TensorforceError.unexpected()
if hasattr(self, function_name):
raise TensorforceError.unexpected()
api_function = getattr(self, attribute)
if not callable(api_function):
raise TensorforceError.unexpected()
function = self.create_api_function(
name='{}.{}'.format(self.name, function_name), api_function=api_function
)
setattr(self, function_name, function)
if self.summarizer_spec is not None:
record_summaries.__exit__(None, None, None)
Module.global_summary_step = None
if self.parent is None:
self.graph_summary = None # TODO!
if self.summarizer_spec is not None:
default_summarizer.__exit__(None, None, None)
def get_module_class_and_kwargs(name,
module,
modules=None,
default_module=None,
**kwargs):
# name
if not util.is_valid_name(name=name):
raise TensorforceError.value(name='module',
argument='name',
value=name)
# module
# ???
# modules
if modules is not None and not isinstance(modules, dict):
raise TensorforceError.type(name='module',
argument='modules',
value=modules)
# default_module
# ???
if isinstance(module, dict):
# Dictionary module specification (type either given via 'type' or 'default_module')
for key, value in module.items():
if key in kwargs and kwargs[key] != value:
raise TensorforceError.mismatch(name='module',
argument=key,
value1=kwargs[key],
value2=value)
kwargs[key] = value
module = kwargs.pop('type', default_module)
return Module.get_module_class_and_kwargs(
name=name,
module=module,
modules=modules,
default_module=default_module,
**kwargs)
elif isinstance(module, str):
if os.path.isfile(module):
# JSON file module specification
with open(module, 'r') as fp:
module = json.load(fp=fp)
return Module.get_module_class_and_kwargs(
name=name,
module=module,
modules=modules,
default_module=default_module,
**kwargs)
elif '.' in module:
# Library module specification
library_name, module_name = module.rsplit('.', 1)
library = importlib.import_module(name=library_name)
module = getattr(library, module_name)
return Module.get_module_class_and_kwargs(
name=name,
module=module,
modules=modules,
default_module=default_module,
**kwargs)
elif modules is not None and module in modules:
# Keyword module specification
return Module.get_module_class_and_kwargs(
name=name,
module=modules[module],
default_module=default_module,
**kwargs)
elif 'default' in modules or default_module is not None:
# Default module specification
if '_first_arg' in kwargs:
raise TensorforceError.value(name='module kwargs',
value='_first_arg')
if module is not None:
kwargs['_first_arg'] = module
if default_module is None:
default_module = modules['default']
return Module.get_module_class_and_kwargs(
name=name,
module=default_module,
modules=modules,
**kwargs)
else:
raise TensorforceError.value(name='module specification',
value=module)
elif not callable(module) and ('default' in modules
or default_module is not None):
# Default module specification
if '_first_arg' in kwargs:
raise TensorforceError.value(name='module kwargs',
value='_first_arg')
if module is not None:
kwargs['_first_arg'] = module
if default_module is None:
default_module = modules['default']
return Module.get_module_class_and_kwargs(name=name,
module=default_module,
modules=modules,
**kwargs)
elif callable(module):
first_arg = kwargs.pop('_first_arg', None)
return module, first_arg, kwargs
else:
raise TensorforceError.value(name='module specification',
value=module)
def add_variable(self,
name,
dtype,
shape,
is_trainable,
initializer='zeros',
summarize=None,
shared=None):
# name
if not util.is_valid_name(name=name):
raise TensorforceError.value(name='variable',
argument='name',
value=name)
elif name in self.variables:
raise TensorforceError.exists(name='variable', value=name)
# dtype
if not util.is_valid_type(dtype=dtype):
raise TensorforceError.value(name='variable',
argument='dtype',
value=dtype)
# shape
if not util.is_iterable(x=shape) or \
not all(isinstance(num_dims, int) for num_dims in shape):
raise TensorforceError.type(name='variable',
argument='shape',
value=shape)
elif not all(num_dims > 0 for num_dims in shape):
raise TensorforceError.value(name='variable',
argument='shape',
value=shape)
# is_trainable
if not isinstance(is_trainable, bool):
raise TensorforceError.type(name='variable',
argument='is_trainable',
value=is_trainable)
# initializer
if not isinstance(initializer, (util.py_dtype(dtype=dtype), np.ndarray, tf.Tensor)) and \
initializer not in ('random', 'zeros', 'ones'):
raise TensorforceError.value(name='variable',
argument='initializer',
value=initializer)
elif isinstance(initializer, np.ndarray) and \
initializer.dtype != util.np_dtype(dtype=dtype):
raise TensorforceError.type(name='variable',
argument='initializer',
value=initializer)
elif isinstance(initializer,
tf.Tensor) and util.dtype(x=initializer) != dtype:
raise TensorforceError.type(name='variable',
argument='initializer',
value=initializer)
elif isinstance(initializer,
str) and initializer == 'random' and dtype != 'float':
raise TensorforceError(
message=
"Invalid variable initializer value for non-float variable: {}."
.format(initializer))
# summarize
if summarize is not None and not isinstance(summarize, bool):
raise TensorforceError.type(name='variable',
argument='summarize',
value=summarize)
# shared
if shared is not None and not isinstance(shared, str):
raise TensorforceError.type(name='variable',
argument='shared',
value=shared)
variable = None
if shared is not None and len(tf.get_collection(key=shared)) > 0:
# Retrieve shared variable from TensorFlow
collection = tf.get_collection(key=shared)
if len(collection) > 1:
raise TensorforceError.unexpected()
variable = collection[0]
else:
tf_dtype = util.tf_dtype(dtype=dtype)
# Variable initializer
if isinstance(initializer, util.py_dtype(dtype=dtype)):
initializer = tf.constant(value=initializer,
dtype=tf_dtype,
shape=shape)
elif isinstance(initializer, np.ndarray):
if initializer.shape != shape:
raise TensorforceError(
"Invalid variable initializer shape: {}.".format(
initializer.shape))
initializer = initializer
elif isinstance(initializer, tf.Tensor):
if util.shape(x=initializer) != shape:
raise TensorforceError(
"Invalid variable initializer shape: {}.".format(
util.shape(x=initializer)))
initializer = initializer
elif not isinstance(initializer, str):
raise TensorforceError(
"Invalid variable initializer: {}".format(initializer))
elif initializer == 'random':
stddev = min(
0.1, sqrt(2.0 / (util.product(xs=shape[:-1]) + shape[-1])))
initializer = tf.random_normal(
shape=shape,
mean=0.0,
stddev=stddev,
dtype=util.tf_dtype(dtype=dtype))
elif initializer == 'zeros':
initializer = tf.zeros(shape=shape, dtype=tf_dtype)
elif initializer == 'ones':
initializer = tf.ones(shape=shape, dtype=tf_dtype)
# Variable
variable = tf.Variable(initial_value=initializer,
trainable=is_trainable,
validate_shape=True,
name=name,
dtype=tf_dtype,
expected_shape=shape) # collections=
# Register shared variable with TensorFlow
if shared is not None:
tf.add_to_collection(name=shared, value=variable)
# Register variable
self.variables[name] = variable
if is_trainable:
self.trainable_variables[name] = variable
# Add summary
if (summarize is None and is_trainable) or summarize:
variable = tf.identity(input=variable)
variable = self.add_summary(label='variables',
name=name,
tensor=variable,
mean_variance=True)
return variable
def initialize(self):
# Check whether module is already initialized
if self.is_initialized:
raise TensorforceError(message="Module already initialized.")
# Set internal attributes
self.is_initialized = True
self.variables = OrderedDict()
self.trainable_variables = OrderedDict()
self.regularized_variables = OrderedDict()
if self.parent is None:
Module.global_scope = list()
Module.global_scope.append(self.name)
# TensorFlow device and variable scope
self.scope = tf.variable_scope(name_or_scope=self.name)
if self.device is not None:
self.device = tf.device(device_name_or_function=self.device)
self.device.__enter__()
with self.scope:
if self.parent is None:
# with tf.device(device_name_or_function=(self.global_model.device if self.global_model else self.device)):
# Global timestep
self.global_timestep = self.add_variable(
name='global-timestep',
dtype='long',
shape=(),
is_trainable=False,
initializer='zeros',
shared='global-timestep')
collection = tf.get_collection(key=tf.GraphKeys.GLOBAL_STEP)
if len(collection) == 0:
tf.add_to_collection(name=tf.GraphKeys.GLOBAL_STEP,
value=self.global_timestep)
# Global episode
self.global_episode = self.add_variable(
name='global-episode',
dtype='long',
shape=(),
is_trainable=False,
initializer='zeros',
shared='global-episode')
Module.global_tensors = OrderedDict(
timestep=self.global_timestep, episode=self.global_episode)
if self.summarizer_spec is not None:
if 'steps' in self.summarizer_spec:
record_summaries = tf.contrib.summary.record_summaries_every_n_global_steps(
n=self.summarizer_spec['steps'],
global_step=self.global_timestep)
else:
record_summaries = tf.contrib.summary.always_record_summaries(
)
record_summaries.__enter__()
for module in self.modules.values():
module.initialize()
self.tf_initialize()
if self.device is not None:
self.device.__exit__(None, None, None)
Module.global_scope.pop()
if self.parent is None:
assert len(Module.global_scope) == 0
Module.global_tensors = None
Module.global_scope = None
# Internal TensorFlow functions, prefixed by 'tf_'
for attribute in sorted(dir(self)):
if attribute.startswith('tf_') and attribute != 'tf_initialize':
function_name = attribute[3:]
if not util.is_valid_name(name=function_name):
raise TensorforceError.value(name='TF-function name',
value=function_name)
if hasattr(self, function_name):
raise TensorforceError.exists(name='TF-function',
value=function_name)
tf_function = getattr(self, attribute)
if not callable(tf_function):
raise TensorforceError.exists(name='TF-function',
value=tf_function)
function = self.create_tf_function(name='{}.{}'.format(
self.name, function_name),
tf_function=tf_function)
setattr(self, function_name, function)
# API TensorFlow functions, prefixed by 'api_'
for attribute in sorted(dir(self)):
if attribute.startswith('api_'):
function_name = attribute[4:]
# Todo: own every_n_step implementation, plus maybe per function steps argument
if function_name == 'observe':
if self.summarizer_spec is not None and 'steps' in self.summarizer_spec:
record_summaries2 = tf.contrib.summary.always_record_summaries(
)
record_summaries2.__enter__()
if not util.is_valid_name(name=function_name):
raise TensorforceError.value(name='API-function name',
value=function_name)
if hasattr(self, function_name):
raise TensorforceError.exists(name='API-function',
value=function_name)
api_function = getattr(self, attribute)
if not callable(api_function):
raise TensorforceError.exists(name='API-function',
value=tf_function)
function = self.create_api_function(name='{}.{}'.format(
self.name, function_name),
api_function=api_function)
setattr(self, function_name, function)
if function_name == 'observe':
if self.summarizer_spec is not None and 'steps' in self.summarizer_spec:
record_summaries2.__exit__(None, None, None)
if self.parent is None:
if self.summarizer_spec is not None:
record_summaries.__exit__(None, None, None)