def __init__(self, variable_scope_name): self._variable_scope_name = variable_scope_name default = variable_scope._get_default_variable_store() # pylint: disable=protected-access if default._store_eager_variables: # pylint: disable=protected-access self._eager_variable_store = variable_scope.EagerVariableStore( default) else: self._eager_variable_store = variable_scope.EagerVariableStore()
def __init__(self, variable_scope_name): self._variable_scope_name = variable_scope_name default = variable_scope._get_default_variable_store() # pylint: disable=protected-access if default._store_eager_variables: # pylint: disable=protected-access self._eager_variable_store = variable_scope.EagerVariableStore(default) else: # If no outer eager variable store has been made, # the template needs to create one self._eager_variable_store = variable_scope.EagerVariableStore() self._used_once = False
def testEagerExecution(self): with context.eager_mode(): container = variable_scope.EagerVariableStore() x = constant_op.constant([[2.0]]) with container.as_default(): y = core_layers.dense( x, 1, name='my_dense', kernel_initializer=init_ops.ones_initializer()) self.assertAllEqual(y, [[2.0]]) self.assertEqual(len(container.variables()), 2) # Recreate the layer to test reuse. with container.as_default(): core_layers.dense( x, 1, name='my_dense', kernel_initializer=init_ops.ones_initializer()) self.assertEqual(len(container.variables()), 2)
def __init__(self, name, func, create_scope_now=False, unique_name=None, custom_getter=None): """Creates a template for the given function. Args: name: A name for the scope created by this template. The name will be made unique by appending `_N` to the it (see how `tf.variable_scope` treats the `default_name` for details). func: The function to apply each time. create_scope_now: Whether to create the scope at Template construction time, rather than first call. Defaults to false. Creating the scope at construction time may be more convenient if the template is passed through much lower level code, and you want to be sure of the scope name without knowing exactly where it will be first called. If set to True, the scope will be created in the constructor, and all subsequent times in __call__, leading to a trailing numeral being added to the names of all created Tensors. If set to False, the scope will be created at the first call location. unique_name: When used, it overrides name_ and is not made unique. If a template of the same scope/unique_name already exists and reuse is false, an error is raised. Defaults to None. custom_getter: optional custom getter to pass to variable_scope() Raises: RuntimeError: if eager mode is not enabled. ValueError: if the name is None or unique_name is provided. """ if not context.in_eager_mode(): raise RuntimeError( "{} objects can only be used when eager execution is enabled, use " "tf.Template for graph construction".format(type(self))) if unique_name: raise ValueError("unique_name cannot be used in eager mode.") super(EagerTemplate, self).__init__(name, func, create_scope_now, unique_name, custom_getter) # Create an eager variable store only if the current variable store cannot # store eager variables. This should allow for correct nesting. default_vstore = variable_scope._get_default_variable_store() # pylint: disable=protected-access if default_vstore._store_eager_variables: # pylint: disable=protected-access raise ValueError( "Nested EagerTemaplates are not currently supported.") else: self._eager_variable_store = variable_scope.EagerVariableStore()
def __init__(self, obs_space, action_space, num_outputs, model_config, name, q_hiddens=(256, ), dueling=False, num_atoms=1, use_noisy=False, v_min=-10.0, v_max=10.0, sigma0=0.5, parameter_noise=False): """Initialize variables of this model. Extra model kwargs: q_hiddens (list): defines size of hidden layers for the q head. These will be used to postprocess the model output for the purposes of computing Q values. dueling (bool): whether to build the state value head for DDQN num_atoms (int): if >1, enables distributional DQN use_noisy (bool): use noisy nets v_min (float): min value support for distributional DQN v_max (float): max value support for distributional DQN sigma0 (float): initial value of noisy nets parameter_noise (bool): enable layer norm for param noise Note that the core layers for forward() are not defined here, this only defines the layers for the Q head. Those layers for forward() should be defined in subclasses of DistributionalQModel. """ super(DistributionalQModel, self).__init__(obs_space, action_space, num_outputs, model_config, name) # setup the Q head output (i.e., model for get_q_values) self.model_out = tf.keras.layers.Input(shape=(num_outputs, ), name="model_out") def build_action_value(model_out): if q_hiddens: action_out = model_out for i in range(len(q_hiddens)): if use_noisy: action_out = self._noisy_layer("hidden_%d" % i, action_out, q_hiddens[i], sigma0) elif parameter_noise: action_out = tf.keras.layers.Dense( units=q_hiddens[i], activation_fn=tf.nn.relu, normalizer_fn=tf.keras.layers.LayerNormalization)( action_out) else: action_out = tf.keras.layers.Dense( units=q_hiddens[i], activation=tf.nn.relu, name="hidden_%d" % i)(action_out) else: # Avoid postprocessing the outputs. This enables custom models # to be used for parametric action DQN. action_out = model_out if use_noisy: action_scores = self._noisy_layer("output", action_out, self.action_space.n * num_atoms, sigma0, non_linear=False) elif q_hiddens: action_scores = tf.keras.layers.Dense( units=self.action_space.n * num_atoms, activation=None)(action_out) else: action_scores = model_out if num_atoms > 1: # Distributional Q-learning uses a discrete support z # to represent the action value distribution z = tf.range(num_atoms, dtype=tf.float32) z = v_min + z * (v_max - v_min) / float(num_atoms - 1) support_logits_per_action = tf.reshape( tensor=action_scores, shape=(-1, self.action_space.n, num_atoms)) support_prob_per_action = tf.nn.softmax( logits=support_logits_per_action) action_scores = tf.reduce_sum(input_tensor=z * support_prob_per_action, axis=-1) logits = support_logits_per_action dist = support_prob_per_action return [ action_scores, z, support_logits_per_action, logits, dist ] else: logits = tf.expand_dims(tf.ones_like(action_scores), -1) dist = tf.expand_dims(tf.ones_like(action_scores), -1) return [action_scores, logits, dist] def build_state_score(model_out): state_out = model_out for i in range(len(q_hiddens)): if use_noisy: state_out = self._noisy_layer("dueling_hidden_%d" % i, state_out, q_hiddens[i], sigma0) elif parameter_noise: state_out = tf.keras.layers.Dense( units=q_hiddens[i], activation_fn=tf.nn.relu, normalizer_fn=tf.contrib.layers.layer_norm)(state_out) else: state_out = tf.keras.layers.Dense( units=q_hiddens[i], activation=tf.nn.relu)(state_out) if use_noisy: state_score = self._noisy_layer("dueling_output", state_out, num_atoms, sigma0, non_linear=False) else: state_score = tf.keras.layers.Dense(units=num_atoms, activation=None)(state_out) return state_score if tf.executing_eagerly(): from tensorflow.python.ops import variable_scope # Have to use a variable store to reuse variables in eager mode store = variable_scope.EagerVariableStore() # Save the scope objects, since in eager we will execute this # path repeatedly and there is no guarantee it will always be run # in the same original scope. with tf.variable_scope(name + "/action_value") as action_scope: pass with tf.variable_scope(name + "/state_value") as state_scope: pass def build_action_value_in_scope(model_out): with store.as_default(): with tf.variable_scope(action_scope, reuse=tf.AUTO_REUSE): return build_action_value(model_out) def build_state_score_in_scope(model_out): with store.as_default(): with tf.variable_scope(state_scope, reuse=tf.AUTO_REUSE): return build_state_score(model_out) else: def build_action_value_in_scope(model_out): with tf.variable_scope(name + "/action_value", reuse=tf.AUTO_REUSE): return build_action_value(model_out) def build_state_score_in_scope(model_out): with tf.variable_scope(name + "/state_value", reuse=tf.AUTO_REUSE): return build_state_score(model_out) q_out = build_action_value_in_scope(self.model_out) self.q_value_head = tf.keras.Model(self.model_out, q_out) self.register_variables(self.q_value_head.variables) if dueling: state_out = build_state_score_in_scope(self.model_out) self.state_value_head = tf.keras.Model(self.model_out, state_out) self.register_variables(self.state_value_head.variables)
def create_eager_var_store(): if context.in_eager_mode(): return variable_scope.EagerVariableStore() else: return DummyVariableStore()
def __init__( self, obs_space, action_space, num_outputs, model_config, name, q_hiddens=(256, ), dueling=False, num_atoms=1, use_noisy=False, v_min=-10.0, v_max=10.0, sigma0=0.5, # TODO(sven): Move `add_layer_norm` into ModelCatalog as # generic option, then error if we use ParameterNoise as # Exploration type and do not have any LayerNorm layers in # the net. add_layer_norm=False): """Initialize variables of this model. Extra model kwargs: q_hiddens (List[int]): List of layer-sizes after(!) the Advantages(A)/Value(V)-split. Hence, each of the A- and V- branches will have this structure of Dense layers. To define the NN before this A/V-split, use - as always - config["model"]["fcnet_hiddens"]. dueling (bool): Whether to build the advantage(A)/value(V) heads for DDQN. If True, Q-values are calculated as: Q = (A - mean[A]) + V. If False, raw NN output is interpreted as Q-values. num_atoms (int): if >1, enables distributional DQN use_noisy (bool): use noisy nets v_min (float): min value support for distributional DQN v_max (float): max value support for distributional DQN sigma0 (float): initial value of noisy nets add_layer_norm (bool): Add a LayerNorm after each layer.. Note that the core layers for forward() are not defined here, this only defines the layers for the Q head. Those layers for forward() should be defined in subclasses of DistributionalQModel. """ super(DistributionalQTFModel, self).__init__( obs_space, action_space, num_outputs, model_config, name) # setup the Q head output (i.e., model for get_q_values) self.model_out = tf.keras.layers.Input( shape=(num_outputs, ), name="model_out") def build_action_value(model_out): if q_hiddens: action_out = model_out for i in range(len(q_hiddens)): if use_noisy: action_out = self._noisy_layer( "hidden_%d" % i, action_out, q_hiddens[i], sigma0) elif add_layer_norm: action_out = tf.keras.layers.Dense( units=q_hiddens[i], activation=tf.nn.relu)(action_out) action_out = \ tf.keras.layers.LayerNormalization()( action_out) else: action_out = tf.keras.layers.Dense( units=q_hiddens[i], activation=tf.nn.relu, name="hidden_%d" % i)(action_out) else: # Avoid postprocessing the outputs. This enables custom models # to be used for parametric action DQN. action_out = model_out if use_noisy: action_scores = self._noisy_layer( "output", action_out, self.action_space.n * num_atoms, sigma0, non_linear=False) elif q_hiddens: action_scores = tf.keras.layers.Dense( units=self.action_space.n * num_atoms, activation=None)(action_out) else: action_scores = model_out if num_atoms > 1: # Distributional Q-learning uses a discrete support z # to represent the action value distribution z = tf.range(num_atoms, dtype=tf.float32) z = v_min + z * (v_max - v_min) / float(num_atoms - 1) def _layer(x): support_logits_per_action = tf.reshape( tensor=x, shape=(-1, self.action_space.n, num_atoms)) support_prob_per_action = tf.nn.softmax( logits=support_logits_per_action) x = tf.reduce_sum( input_tensor=z * support_prob_per_action, axis=-1) logits = support_logits_per_action dist = support_prob_per_action return [x, z, support_logits_per_action, logits, dist] return tf.keras.layers.Lambda(_layer)(action_scores) else: logits = tf.expand_dims(tf.ones_like(action_scores), -1) dist = tf.expand_dims(tf.ones_like(action_scores), -1) return [action_scores, logits, dist] def build_state_score(model_out): state_out = model_out for i in range(len(q_hiddens)): if use_noisy: state_out = self._noisy_layer("dueling_hidden_%d" % i, state_out, q_hiddens[i], sigma0) else: state_out = tf.keras.layers.Dense( units=q_hiddens[i], activation=tf.nn.relu)(state_out) if add_layer_norm: state_out = tf.keras.layers.LayerNormalization()( state_out) if use_noisy: state_score = self._noisy_layer( "dueling_output", state_out, num_atoms, sigma0, non_linear=False) else: state_score = tf.keras.layers.Dense( units=num_atoms, activation=None)(state_out) return state_score if tf.executing_eagerly(): from tensorflow.python.ops import variable_scope # Have to use a variable store to reuse variables in eager mode store = variable_scope.EagerVariableStore() # Save the scope objects, since in eager we will execute this # path repeatedly and there is no guarantee it will always be run # in the same original scope. with tf.variable_scope(name + "/action_value") as action_scope: pass with tf.variable_scope(name + "/state_value") as state_scope: pass def build_action_value_in_scope(model_out): with store.as_default(): with tf.variable_scope(action_scope, reuse=tf.AUTO_REUSE): return build_action_value(model_out) def build_state_score_in_scope(model_out): with store.as_default(): with tf.variable_scope(state_scope, reuse=tf.AUTO_REUSE): return build_state_score(model_out) else: def build_action_value_in_scope(model_out): with tf.variable_scope( name + "/action_value", reuse=tf.AUTO_REUSE): return build_action_value(model_out) def build_state_score_in_scope(model_out): with tf.variable_scope( name + "/state_value", reuse=tf.AUTO_REUSE): return build_state_score(model_out) q_out = build_action_value_in_scope(self.model_out) self.q_value_head = tf.keras.Model(self.model_out, q_out) self.register_variables(self.q_value_head.variables) if dueling: state_out = build_state_score_in_scope(self.model_out) self.state_value_head = tf.keras.Model(self.model_out, state_out) self.register_variables(self.state_value_head.variables)
def main(): FLAGS = Args() # Enable TF Eager execution tfe = tf.contrib.eager tfe.enable_eager_execution() # sample sentence input_str = 'Twas brillig, and the slithy toves Did gyre and gimble in the wade; All mimsy were the borogoves, And the mome raths outgrabe.' # convert sentence into index in vocab wmt_problem = problems.problem(FLAGS.problem) encoders = wmt_problem.feature_encoders(FLAGS.data_dir) inputs = encoders["inputs"].encode(input_str) + [1] # add EOS id batch_inputs = tf.reshape(inputs, [1, -1, 1]) # Make it 3D. features = {"inputs": batch_inputs} # initialize translation model hparams_set = FLAGS.hparams_set Modes = tf.estimator.ModeKeys hparams = trainer_lib.create_hparams(hparams_set, data_dir=FLAGS.data_dir, problem_name=FLAGS.problem) translate_model = registry.model(FLAGS.model)(hparams, Modes.EVAL) # recover parameters and conduct recurrent conduction ckpt_dir = tf.train.latest_checkpoint(FLAGS.model_dir) with tfe.restore_variables_on_create(ckpt_dir): with variable_scope.EagerVariableStore().as_default(): with tf.variable_scope('universal_transformer'): # Convert word index to word embedding features = translate_model.bottom(features) with tf.variable_scope('universal_transformer/body'): input_tensor = tf.convert_to_tensor(features['inputs']) input_tensor = common_layers.flatten4d3d(input_tensor) encoder_input, self_attention_bias, _ = ( transformer.transformer_prepare_encoder( input_tensor, tf.convert_to_tensor([0]), translate_model.hparams, features=None)) with tf.variable_scope('universal_transformer/body/encoder'): ffn_unit = functools.partial( universal_transformer_util.transformer_encoder_ffn_unit, hparams=translate_model.hparams) attention_unit = functools.partial( universal_transformer_util.transformer_encoder_attention_unit, hparams=translate_model.hparams, encoder_self_attention_bias=None, attention_dropout_broadcast_dims=[], save_weights_to={}, make_image_summary=True) storing_list = [] transformed_state = encoder_input for step_index in range(1024): storing_list.append(transformed_state.numpy()) with tf.variable_scope('universal_transformer/body/encoder/universal_transformer_{}'.format(FLAGS.ut_type)): transformed_state = universal_transformer_util.step_preprocess( transformed_state, tf.convert_to_tensor(step_index % FLAGS.step_num), translate_model.hparams ) with tf.variable_scope('universal_transformer/body/encoder/universal_transformer_{}/rec_layer_0'.format(FLAGS.ut_type)): transformed_new_state = ffn_unit(attention_unit(transformed_state)) with tf.variable_scope('universal_transformer/body/encoder'): if (step_index + 1) % FLAGS.step_num == 0: transformed_new_state = common_layers.layer_preprocess(transformed_new_state, translate_model.hparams) if step_index == 5: print(transformed_new_state) transformed_state = transformed_new_state storing_list = np.asarray(storing_list) np.save(FLAGS.save_dir, storing_list)
def main(): FLAGS = Args() # Enable TF Eager execution tfe = tf.contrib.eager tfe.enable_eager_execution() batch_inputs = input_generator() # initialize translation model hparams_set = FLAGS.hparams_set Modes = tf.estimator.ModeKeys hparams = trainer_lib.create_hparams(hparams_set, data_dir=FLAGS.data_dir, problem_name=FLAGS.problem) translate_model = registry.model(FLAGS.model)(hparams, Modes.EVAL) # recover parameters and conduct recurrent conduction ckpt_dir = tf.train.latest_checkpoint(FLAGS.model_dir) with tfe.restore_variables_on_create(ckpt_dir): with variable_scope.EagerVariableStore().as_default(): features = {'inputs': batch_inputs} with tf.variable_scope('universal_transformer/body'): input_tensor = tf.convert_to_tensor(features['inputs']) input_tensor = common_layers.flatten4d3d(input_tensor) encoder_input, self_attention_bias, _ = ( transformer.transformer_prepare_encoder( input_tensor, tf.convert_to_tensor([0]), translate_model.hparams, features=None)) with tf.variable_scope('universal_transformer/body/encoder'): ffn_unit = functools.partial( universal_transformer_util.transformer_encoder_ffn_unit, hparams=translate_model.hparams) attention_unit = functools.partial( universal_transformer_util. transformer_encoder_attention_unit, hparams=translate_model.hparams, encoder_self_attention_bias=None, attention_dropout_broadcast_dims=[], save_weights_to={}, make_image_summary=True) storing_list = [] transformed_state = encoder_input for step_index in range(1024): storing_list.append(transformed_state.numpy()) with tf.variable_scope( 'universal_transformer/body/encoder/universal_transformer_{}' .format(FLAGS.ut_type)): transformed_state = universal_transformer_util.step_preprocess( transformed_state, tf.convert_to_tensor(step_index % FLAGS.step_num), translate_model.hparams) with tf.variable_scope( 'universal_transformer/body/encoder/universal_transformer_{}/rec_layer_0' .format(FLAGS.ut_type)): transformed_new_state = ffn_unit( attention_unit(transformed_state)) with tf.variable_scope('universal_transformer/body/encoder'): if (step_index + 1) % FLAGS.step_num == 0: transformed_new_state = common_layers.layer_preprocess( transformed_new_state, translate_model.hparams) if step_index == 5: print(transformed_new_state) transformed_state = transformed_new_state storing_list = np.asarray(storing_list) np.save(FLAGS.save_dir, storing_list)
#tf.compat.v1.enable_eager_execution() print("TensorFlow 版本: {}".format(tf.version.VERSION)) print("Eager execution: {}".format(tf.executing_eagerly())) #生成模拟数据 train_X = np.linspace(-1, 1, 100) train_Y = 2 * train_X + np.random.randn(*train_X.shape) * 0.3 # y=2x,但是加入了噪声 from tensorflow.python.ops import variable_scope #建立数据集 dataset = tf.data.Dataset.from_tensor_slices( (np.reshape(train_X,[-1,1]),np.reshape(train_X,[-1,1])) ) dataset = dataset.repeat().batch(1) global_step = tf.compat.v1.train.get_or_create_global_step() container = variable_scope.EagerVariableStore()#container进行了改变 learning_rate = 0.01 # 随机梯度下降法作为优化器 optimizer = tf.compat.v1.train.GradientDescentOptimizer(learning_rate=learning_rate) def getcost(x,y):#定义函数,计算loss值 # 前向结构 with container.as_default():#将动态图使用的层包装起来。可以得到变量 # z = tf.contrib.slim.fully_connected(x, 1,reuse=tf.AUTO_REUSE) z = tf.compat.v1.layers.dense(x,1, name="l1") cost =tf.reduce_mean( input_tensor=tf.square(y - z))#loss值 return cost def grad( inputs, targets): with tf.GradientTape() as tape: