def __init__(self, variable_scope_name):
self._variable_scope_name = variable_scope_name
default = variable_scope._get_default_variable_store() # pylint: disable=protected-access
if default._store_eager_variables: # pylint: disable=protected-access
self._eager_variable_store = variable_scope.EagerVariableStore(
default)
else:
self._eager_variable_store = variable_scope.EagerVariableStore()
def __init__(self, variable_scope_name):
self._variable_scope_name = variable_scope_name
default = variable_scope._get_default_variable_store() # pylint: disable=protected-access
if default._store_eager_variables: # pylint: disable=protected-access
self._eager_variable_store = variable_scope.EagerVariableStore(default)
else:
# If no outer eager variable store has been made,
# the template needs to create one
self._eager_variable_store = variable_scope.EagerVariableStore()
self._used_once = False
def testEagerExecution(self):
with context.eager_mode():
container = variable_scope.EagerVariableStore()
x = constant_op.constant([[2.0]])
with container.as_default():
y = core_layers.dense(
x, 1, name='my_dense',
kernel_initializer=init_ops.ones_initializer())
self.assertAllEqual(y, [[2.0]])
self.assertEqual(len(container.variables()), 2)
# Recreate the layer to test reuse.
with container.as_default():
core_layers.dense(
x, 1, name='my_dense',
kernel_initializer=init_ops.ones_initializer())
self.assertEqual(len(container.variables()), 2)
def __init__(self,
name,
func,
create_scope_now=False,
unique_name=None,
custom_getter=None):
"""Creates a template for the given function.
Args:
name: A name for the scope created by this template. The
name will be made unique by appending `_N` to the it (see how
`tf.variable_scope` treats the `default_name` for details).
func: The function to apply each time.
create_scope_now: Whether to create the scope at Template construction
time, rather than first call. Defaults to false. Creating the scope at
construction time may be more convenient if the template is passed
through much lower level code, and you want to be sure of the scope
name without knowing exactly where it will be first called. If set to
True, the scope will be created in the constructor, and all subsequent
times in __call__, leading to a trailing numeral being added to the
names of all created Tensors. If set to False, the scope will be created
at the first call location.
unique_name: When used, it overrides name_ and is not made unique. If a
template of the same scope/unique_name already exists and reuse is
false, an error is raised. Defaults to None.
custom_getter: optional custom getter to pass to variable_scope()
Raises:
RuntimeError: if eager mode is not enabled.
ValueError: if the name is None or unique_name is provided.
"""
if not context.in_eager_mode():
raise RuntimeError(
"{} objects can only be used when eager execution is enabled, use "
"tf.Template for graph construction".format(type(self)))
if unique_name:
raise ValueError("unique_name cannot be used in eager mode.")
super(EagerTemplate, self).__init__(name, func, create_scope_now,
unique_name, custom_getter)
# Create an eager variable store only if the current variable store cannot
# store eager variables. This should allow for correct nesting.
default_vstore = variable_scope._get_default_variable_store() # pylint: disable=protected-access
if default_vstore._store_eager_variables: # pylint: disable=protected-access
raise ValueError(
"Nested EagerTemaplates are not currently supported.")
else:
self._eager_variable_store = variable_scope.EagerVariableStore()
def __init__(self,
obs_space,
action_space,
num_outputs,
model_config,
name,
q_hiddens=(256, ),
dueling=False,
num_atoms=1,
use_noisy=False,
v_min=-10.0,
v_max=10.0,
sigma0=0.5,
parameter_noise=False):
"""Initialize variables of this model.
Extra model kwargs:
q_hiddens (list): defines size of hidden layers for the q head.
These will be used to postprocess the model output for the
purposes of computing Q values.
dueling (bool): whether to build the state value head for DDQN
num_atoms (int): if >1, enables distributional DQN
use_noisy (bool): use noisy nets
v_min (float): min value support for distributional DQN
v_max (float): max value support for distributional DQN
sigma0 (float): initial value of noisy nets
parameter_noise (bool): enable layer norm for param noise
Note that the core layers for forward() are not defined here, this
only defines the layers for the Q head. Those layers for forward()
should be defined in subclasses of DistributionalQModel.
"""
super(DistributionalQModel,
self).__init__(obs_space, action_space, num_outputs,
model_config, name)
# setup the Q head output (i.e., model for get_q_values)
self.model_out = tf.keras.layers.Input(shape=(num_outputs, ),
name="model_out")
def build_action_value(model_out):
if q_hiddens:
action_out = model_out
for i in range(len(q_hiddens)):
if use_noisy:
action_out = self._noisy_layer("hidden_%d" % i,
action_out,
q_hiddens[i], sigma0)
elif parameter_noise:
action_out = tf.keras.layers.Dense(
units=q_hiddens[i],
activation_fn=tf.nn.relu,
normalizer_fn=tf.keras.layers.LayerNormalization)(
action_out)
else:
action_out = tf.keras.layers.Dense(
units=q_hiddens[i],
activation=tf.nn.relu,
name="hidden_%d" % i)(action_out)
else:
# Avoid postprocessing the outputs. This enables custom models
# to be used for parametric action DQN.
action_out = model_out
if use_noisy:
action_scores = self._noisy_layer("output",
action_out,
self.action_space.n *
num_atoms,
sigma0,
non_linear=False)
elif q_hiddens:
action_scores = tf.keras.layers.Dense(
units=self.action_space.n * num_atoms,
activation=None)(action_out)
else:
action_scores = model_out
if num_atoms > 1:
# Distributional Q-learning uses a discrete support z
# to represent the action value distribution
z = tf.range(num_atoms, dtype=tf.float32)
z = v_min + z * (v_max - v_min) / float(num_atoms - 1)
support_logits_per_action = tf.reshape(
tensor=action_scores,
shape=(-1, self.action_space.n, num_atoms))
support_prob_per_action = tf.nn.softmax(
logits=support_logits_per_action)
action_scores = tf.reduce_sum(input_tensor=z *
support_prob_per_action,
axis=-1)
logits = support_logits_per_action
dist = support_prob_per_action
return [
action_scores, z, support_logits_per_action, logits, dist
]
else:
logits = tf.expand_dims(tf.ones_like(action_scores), -1)
dist = tf.expand_dims(tf.ones_like(action_scores), -1)
return [action_scores, logits, dist]
def build_state_score(model_out):
state_out = model_out
for i in range(len(q_hiddens)):
if use_noisy:
state_out = self._noisy_layer("dueling_hidden_%d" % i,
state_out, q_hiddens[i],
sigma0)
elif parameter_noise:
state_out = tf.keras.layers.Dense(
units=q_hiddens[i],
activation_fn=tf.nn.relu,
normalizer_fn=tf.contrib.layers.layer_norm)(state_out)
else:
state_out = tf.keras.layers.Dense(
units=q_hiddens[i], activation=tf.nn.relu)(state_out)
if use_noisy:
state_score = self._noisy_layer("dueling_output",
state_out,
num_atoms,
sigma0,
non_linear=False)
else:
state_score = tf.keras.layers.Dense(units=num_atoms,
activation=None)(state_out)
return state_score
if tf.executing_eagerly():
from tensorflow.python.ops import variable_scope
# Have to use a variable store to reuse variables in eager mode
store = variable_scope.EagerVariableStore()
# Save the scope objects, since in eager we will execute this
# path repeatedly and there is no guarantee it will always be run
# in the same original scope.
with tf.variable_scope(name + "/action_value") as action_scope:
pass
with tf.variable_scope(name + "/state_value") as state_scope:
pass
def build_action_value_in_scope(model_out):
with store.as_default():
with tf.variable_scope(action_scope, reuse=tf.AUTO_REUSE):
return build_action_value(model_out)
def build_state_score_in_scope(model_out):
with store.as_default():
with tf.variable_scope(state_scope, reuse=tf.AUTO_REUSE):
return build_state_score(model_out)
else:
def build_action_value_in_scope(model_out):
with tf.variable_scope(name + "/action_value",
reuse=tf.AUTO_REUSE):
return build_action_value(model_out)
def build_state_score_in_scope(model_out):
with tf.variable_scope(name + "/state_value",
reuse=tf.AUTO_REUSE):
return build_state_score(model_out)
q_out = build_action_value_in_scope(self.model_out)
self.q_value_head = tf.keras.Model(self.model_out, q_out)
self.register_variables(self.q_value_head.variables)
if dueling:
state_out = build_state_score_in_scope(self.model_out)
self.state_value_head = tf.keras.Model(self.model_out, state_out)
self.register_variables(self.state_value_head.variables)
def create_eager_var_store():
if context.in_eager_mode():
return variable_scope.EagerVariableStore()
else:
return DummyVariableStore()
def __init__(
self,
obs_space,
action_space,
num_outputs,
model_config,
name,
q_hiddens=(256, ),
dueling=False,
num_atoms=1,
use_noisy=False,
v_min=-10.0,
v_max=10.0,
sigma0=0.5,
# TODO(sven): Move `add_layer_norm` into ModelCatalog as
# generic option, then error if we use ParameterNoise as
# Exploration type and do not have any LayerNorm layers in
# the net.
add_layer_norm=False):
"""Initialize variables of this model.
Extra model kwargs:
q_hiddens (List[int]): List of layer-sizes after(!) the
Advantages(A)/Value(V)-split. Hence, each of the A- and V-
branches will have this structure of Dense layers. To define
the NN before this A/V-split, use - as always -
config["model"]["fcnet_hiddens"].
dueling (bool): Whether to build the advantage(A)/value(V) heads
for DDQN. If True, Q-values are calculated as:
Q = (A - mean[A]) + V. If False, raw NN output is interpreted
as Q-values.
num_atoms (int): if >1, enables distributional DQN
use_noisy (bool): use noisy nets
v_min (float): min value support for distributional DQN
v_max (float): max value support for distributional DQN
sigma0 (float): initial value of noisy nets
add_layer_norm (bool): Add a LayerNorm after each layer..
Note that the core layers for forward() are not defined here, this
only defines the layers for the Q head. Those layers for forward()
should be defined in subclasses of DistributionalQModel.
"""
super(DistributionalQTFModel, self).__init__(
obs_space, action_space, num_outputs, model_config, name)
# setup the Q head output (i.e., model for get_q_values)
self.model_out = tf.keras.layers.Input(
shape=(num_outputs, ), name="model_out")
def build_action_value(model_out):
if q_hiddens:
action_out = model_out
for i in range(len(q_hiddens)):
if use_noisy:
action_out = self._noisy_layer(
"hidden_%d" % i, action_out, q_hiddens[i], sigma0)
elif add_layer_norm:
action_out = tf.keras.layers.Dense(
units=q_hiddens[i],
activation=tf.nn.relu)(action_out)
action_out = \
tf.keras.layers.LayerNormalization()(
action_out)
else:
action_out = tf.keras.layers.Dense(
units=q_hiddens[i],
activation=tf.nn.relu,
name="hidden_%d" % i)(action_out)
else:
# Avoid postprocessing the outputs. This enables custom models
# to be used for parametric action DQN.
action_out = model_out
if use_noisy:
action_scores = self._noisy_layer(
"output",
action_out,
self.action_space.n * num_atoms,
sigma0,
non_linear=False)
elif q_hiddens:
action_scores = tf.keras.layers.Dense(
units=self.action_space.n * num_atoms,
activation=None)(action_out)
else:
action_scores = model_out
if num_atoms > 1:
# Distributional Q-learning uses a discrete support z
# to represent the action value distribution
z = tf.range(num_atoms, dtype=tf.float32)
z = v_min + z * (v_max - v_min) / float(num_atoms - 1)
def _layer(x):
support_logits_per_action = tf.reshape(
tensor=x, shape=(-1, self.action_space.n, num_atoms))
support_prob_per_action = tf.nn.softmax(
logits=support_logits_per_action)
x = tf.reduce_sum(
input_tensor=z * support_prob_per_action, axis=-1)
logits = support_logits_per_action
dist = support_prob_per_action
return [x, z, support_logits_per_action, logits, dist]
return tf.keras.layers.Lambda(_layer)(action_scores)
else:
logits = tf.expand_dims(tf.ones_like(action_scores), -1)
dist = tf.expand_dims(tf.ones_like(action_scores), -1)
return [action_scores, logits, dist]
def build_state_score(model_out):
state_out = model_out
for i in range(len(q_hiddens)):
if use_noisy:
state_out = self._noisy_layer("dueling_hidden_%d" % i,
state_out, q_hiddens[i],
sigma0)
else:
state_out = tf.keras.layers.Dense(
units=q_hiddens[i], activation=tf.nn.relu)(state_out)
if add_layer_norm:
state_out = tf.keras.layers.LayerNormalization()(
state_out)
if use_noisy:
state_score = self._noisy_layer(
"dueling_output",
state_out,
num_atoms,
sigma0,
non_linear=False)
else:
state_score = tf.keras.layers.Dense(
units=num_atoms, activation=None)(state_out)
return state_score
if tf.executing_eagerly():
from tensorflow.python.ops import variable_scope
# Have to use a variable store to reuse variables in eager mode
store = variable_scope.EagerVariableStore()
# Save the scope objects, since in eager we will execute this
# path repeatedly and there is no guarantee it will always be run
# in the same original scope.
with tf.variable_scope(name + "/action_value") as action_scope:
pass
with tf.variable_scope(name + "/state_value") as state_scope:
pass
def build_action_value_in_scope(model_out):
with store.as_default():
with tf.variable_scope(action_scope, reuse=tf.AUTO_REUSE):
return build_action_value(model_out)
def build_state_score_in_scope(model_out):
with store.as_default():
with tf.variable_scope(state_scope, reuse=tf.AUTO_REUSE):
return build_state_score(model_out)
else:
def build_action_value_in_scope(model_out):
with tf.variable_scope(
name + "/action_value", reuse=tf.AUTO_REUSE):
return build_action_value(model_out)
def build_state_score_in_scope(model_out):
with tf.variable_scope(
name + "/state_value", reuse=tf.AUTO_REUSE):
return build_state_score(model_out)
q_out = build_action_value_in_scope(self.model_out)
self.q_value_head = tf.keras.Model(self.model_out, q_out)
self.register_variables(self.q_value_head.variables)
if dueling:
state_out = build_state_score_in_scope(self.model_out)
self.state_value_head = tf.keras.Model(self.model_out, state_out)
self.register_variables(self.state_value_head.variables)
def main():
FLAGS = Args()
# Enable TF Eager execution
tfe = tf.contrib.eager
tfe.enable_eager_execution()
# sample sentence
input_str = 'Twas brillig, and the slithy toves Did gyre and gimble in the wade; All mimsy were the borogoves, And the mome raths outgrabe.'
# convert sentence into index in vocab
wmt_problem = problems.problem(FLAGS.problem)
encoders = wmt_problem.feature_encoders(FLAGS.data_dir)
inputs = encoders["inputs"].encode(input_str) + [1] # add EOS id
batch_inputs = tf.reshape(inputs, [1, -1, 1]) # Make it 3D.
features = {"inputs": batch_inputs}
# initialize translation model
hparams_set = FLAGS.hparams_set
Modes = tf.estimator.ModeKeys
hparams = trainer_lib.create_hparams(hparams_set, data_dir=FLAGS.data_dir, problem_name=FLAGS.problem)
translate_model = registry.model(FLAGS.model)(hparams, Modes.EVAL)
# recover parameters and conduct recurrent conduction
ckpt_dir = tf.train.latest_checkpoint(FLAGS.model_dir)
with tfe.restore_variables_on_create(ckpt_dir):
with variable_scope.EagerVariableStore().as_default():
with tf.variable_scope('universal_transformer'):
# Convert word index to word embedding
features = translate_model.bottom(features)
with tf.variable_scope('universal_transformer/body'):
input_tensor = tf.convert_to_tensor(features['inputs'])
input_tensor = common_layers.flatten4d3d(input_tensor)
encoder_input, self_attention_bias, _ = (
transformer.transformer_prepare_encoder(
input_tensor, tf.convert_to_tensor([0]), translate_model.hparams, features=None))
with tf.variable_scope('universal_transformer/body/encoder'):
ffn_unit = functools.partial(
universal_transformer_util.transformer_encoder_ffn_unit,
hparams=translate_model.hparams)
attention_unit = functools.partial(
universal_transformer_util.transformer_encoder_attention_unit,
hparams=translate_model.hparams,
encoder_self_attention_bias=None,
attention_dropout_broadcast_dims=[],
save_weights_to={},
make_image_summary=True)
storing_list = []
transformed_state = encoder_input
for step_index in range(1024):
storing_list.append(transformed_state.numpy())
with tf.variable_scope('universal_transformer/body/encoder/universal_transformer_{}'.format(FLAGS.ut_type)):
transformed_state = universal_transformer_util.step_preprocess(
transformed_state,
tf.convert_to_tensor(step_index % FLAGS.step_num),
translate_model.hparams
)
with tf.variable_scope('universal_transformer/body/encoder/universal_transformer_{}/rec_layer_0'.format(FLAGS.ut_type)):
transformed_new_state = ffn_unit(attention_unit(transformed_state))
with tf.variable_scope('universal_transformer/body/encoder'):
if (step_index + 1) % FLAGS.step_num == 0:
transformed_new_state = common_layers.layer_preprocess(transformed_new_state, translate_model.hparams)
if step_index == 5:
print(transformed_new_state)
transformed_state = transformed_new_state
storing_list = np.asarray(storing_list)
np.save(FLAGS.save_dir, storing_list)
def main():
FLAGS = Args()
# Enable TF Eager execution
tfe = tf.contrib.eager
tfe.enable_eager_execution()
batch_inputs = input_generator()
# initialize translation model
hparams_set = FLAGS.hparams_set
Modes = tf.estimator.ModeKeys
hparams = trainer_lib.create_hparams(hparams_set,
data_dir=FLAGS.data_dir,
problem_name=FLAGS.problem)
translate_model = registry.model(FLAGS.model)(hparams, Modes.EVAL)
# recover parameters and conduct recurrent conduction
ckpt_dir = tf.train.latest_checkpoint(FLAGS.model_dir)
with tfe.restore_variables_on_create(ckpt_dir):
with variable_scope.EagerVariableStore().as_default():
features = {'inputs': batch_inputs}
with tf.variable_scope('universal_transformer/body'):
input_tensor = tf.convert_to_tensor(features['inputs'])
input_tensor = common_layers.flatten4d3d(input_tensor)
encoder_input, self_attention_bias, _ = (
transformer.transformer_prepare_encoder(
input_tensor,
tf.convert_to_tensor([0]),
translate_model.hparams,
features=None))
with tf.variable_scope('universal_transformer/body/encoder'):
ffn_unit = functools.partial(
universal_transformer_util.transformer_encoder_ffn_unit,
hparams=translate_model.hparams)
attention_unit = functools.partial(
universal_transformer_util.
transformer_encoder_attention_unit,
hparams=translate_model.hparams,
encoder_self_attention_bias=None,
attention_dropout_broadcast_dims=[],
save_weights_to={},
make_image_summary=True)
storing_list = []
transformed_state = encoder_input
for step_index in range(1024):
storing_list.append(transformed_state.numpy())
with tf.variable_scope(
'universal_transformer/body/encoder/universal_transformer_{}'
.format(FLAGS.ut_type)):
transformed_state = universal_transformer_util.step_preprocess(
transformed_state,
tf.convert_to_tensor(step_index % FLAGS.step_num),
translate_model.hparams)
with tf.variable_scope(
'universal_transformer/body/encoder/universal_transformer_{}/rec_layer_0'
.format(FLAGS.ut_type)):
transformed_new_state = ffn_unit(
attention_unit(transformed_state))
with tf.variable_scope('universal_transformer/body/encoder'):
if (step_index + 1) % FLAGS.step_num == 0:
transformed_new_state = common_layers.layer_preprocess(
transformed_new_state, translate_model.hparams)
if step_index == 5:
print(transformed_new_state)
transformed_state = transformed_new_state
storing_list = np.asarray(storing_list)
np.save(FLAGS.save_dir, storing_list)
#tf.compat.v1.enable_eager_execution()
print("TensorFlow 版本: {}".format(tf.version.VERSION))
print("Eager execution: {}".format(tf.executing_eagerly()))
#生成模拟数据
train_X = np.linspace(-1, 1, 100)
train_Y = 2 * train_X + np.random.randn(*train_X.shape) * 0.3 # y=2x，但是加入了噪声
from tensorflow.python.ops import variable_scope
#建立数据集
dataset = tf.data.Dataset.from_tensor_slices( (np.reshape(train_X,[-1,1]),np.reshape(train_X,[-1,1])) )
dataset = dataset.repeat().batch(1)
global_step = tf.compat.v1.train.get_or_create_global_step()
container = variable_scope.EagerVariableStore()#container进行了改变
learning_rate = 0.01
# 随机梯度下降法作为优化器
optimizer = tf.compat.v1.train.GradientDescentOptimizer(learning_rate=learning_rate)
def getcost(x,y):#定义函数，计算loss值
# 前向结构
with container.as_default():#将动态图使用的层包装起来。可以得到变量
# z = tf.contrib.slim.fully_connected(x, 1,reuse=tf.AUTO_REUSE)
z = tf.compat.v1.layers.dense(x,1, name="l1")
cost =tf.reduce_mean( input_tensor=tf.square(y - z))#loss值
return cost
def grad( inputs, targets):
with tf.GradientTape() as tape:
