def variables_to_restore(self, moving_avg_variables=None): """[Designed for TF 1.x] Returns a map of names to `Variables` to restore. (Designed to work with legacy `tf.compat.v1.train.Saver`, sensitive to specific variable names and not recommended for TF2) If a variable has a moving average, use the moving average variable name as the restore name; otherwise, use the variable name. For example, ```python variables_to_restore = ema.variables_to_restore() saver = tf.compat.v1.train.Saver(variables_to_restore) ``` Below is an example of such mapping: ``` conv/batchnorm/gamma/ExponentialMovingAverage: conv/batchnorm/gamma, conv_4/conv2d_params/ExponentialMovingAverage: conv_4/conv2d_params, global_step: global_step ``` Args: moving_avg_variables: a list of variables that require to use of the moving average variable name to be restored. If None, it will default to variables.moving_average_variables() + variables.trainable_variables() Returns: A map from restore_names to variables. The restore_name is either the original or the moving average version of the variable name, depending on whether the variable name is in the `moving_avg_variables`. """ name_map = {} if moving_avg_variables is None: # Include trainable variables and variables which have been explicitly # added to the moving_average_variables collection. moving_avg_variables = variables.trainable_variables() moving_avg_variables += variables.moving_average_variables() # Remove duplicates moving_avg_variables = set(v.ref() for v in moving_avg_variables) # Collect all the variables with moving average, for v in moving_avg_variables: name_map[self.average_name(v.deref())] = v.deref() # Make sure we restore variables without moving averages as well. moving_avg_variable_names = set( v.deref().name for v in moving_avg_variables) for v in list(set(variables.global_variables())): if v.name not in moving_avg_variable_names and v.op.name not in name_map: name_map[v.op.name] = v return name_map
def variables_to_restore(self, moving_avg_variables=None): """Returns a map of names to `Variables` to restore. If a variable has a moving average, use the moving average variable name as the restore name; otherwise, use the variable name. For example, ```python variables_to_restore = ema.variables_to_restore() saver = tf.compat.v1.train.Saver(variables_to_restore) ``` Below is an example of such mapping: ``` conv/batchnorm/gamma/ExponentialMovingAverage: conv/batchnorm/gamma, conv_4/conv2d_params/ExponentialMovingAverage: conv_4/conv2d_params, global_step: global_step ``` Args: moving_avg_variables: a list of variables that require to use of the moving average variable name to be restored. If None, it will default to variables.moving_average_variables() + variables.trainable_variables() Returns: A map from restore_names to variables. The restore_name is either the original or the moving average version of the variable name, depending on whether the variable name is in the `moving_avg_variables`. """ name_map = {} if moving_avg_variables is None: # Include trainable variables and variables which have been explicitly # added to the moving_average_variables collection. moving_avg_variables = variables.trainable_variables() moving_avg_variables += variables.moving_average_variables() # Remove duplicates moving_avg_variables = set(moving_avg_variables) # Collect all the variables with moving average, for v in moving_avg_variables: name_map[self.average_name(v)] = v # Make sure we restore variables without moving averages as well. moving_avg_variable_names = set([v.name for v in moving_avg_variables]) for v in list(set(variables.global_variables())): if v.name not in moving_avg_variable_names and v.op.name not in name_map: name_map[v.op.name] = v return name_map
def variables_to_restore(self, moving_avg_variables=None): """Returns a map of names to `Variables` to restore. If a variable has a moving average, use the moving average variable name as the restore name; otherwise, use the variable name. For example, ```python variables_to_restore = ema.variables_to_restore() saver = tf.train.Saver(variables_to_restore) ``` Below is an example of such mapping: ``` conv/batchnorm/gamma/ExponentialMovingAverage: conv/batchnorm/gamma, conv_4/conv2d_params/ExponentialMovingAverage: conv_4/conv2d_params, global_step: global_step ``` Args: moving_avg_variables: a list of variables that require to use of the moving variable name to be restored. If None, it will default to variables.moving_average_variables() + variables.trainable_variables() Returns: A map from restore_names to variables. The restore_name can be the moving_average version of the variable name if it exist, or the original variable name. """ name_map = {} if moving_avg_variables is None: # Include trainable variables and variables which have been explicitly # added to the moving_average_variables collection. moving_avg_variables = variables.trainable_variables() moving_avg_variables += variables.moving_average_variables() # Remove duplicates moving_avg_variables = set(moving_avg_variables) # Collect all the variables with moving average, for v in moving_avg_variables: name_map[self.average_name(v)] = v # Make sure we restore variables without moving averages as well. moving_avg_variable_names = set([v.name for v in moving_avg_variables]) for v in list(set(variables.global_variables())): if v.name not in moving_avg_variable_names and v.op.name not in name_map: name_map[v.op.name] = v return name_map
def variables_to_restore(self, moving_avg_variables=None): """""" name_map = {} if moving_avg_variables is None: moving_avg_variables = variables.trainable_variables() moving_avg_variables += variables.moving_average_variables() # Remove duplicates moving_avg_variables = set(moving_avg_variables) # Collect all the variables with moving average, for v in moving_avg_variables: name_map[self.average_name(v)] = v # Make sure we restore variables without moving average as well. for v in list(set(variables.all_variables()) - moving_avg_variables): if v.op.name not in name_map: name_map[v.op.name] = v return name_map
def variables_to_restore(self): """Returns a map of names to `Variables` to restore. If a variable has a moving average, use the moving average variable name as the restore name; otherwise, use the variable name. For example, ```python variables_to_restore = ema.variables_to_restore() saver = tf.train.Saver(variables_to_restore) ``` Below is an example of such mapping: ``` conv/batchnorm/gamma/ExponentialMovingAverage: conv/batchnorm/gamma, conv_4/conv2d_params/ExponentialMovingAverage: conv_4/conv2d_params, global_step: global_step ``` Returns: A map from restore_names to variables. The restore_name can be the moving_average version of the variable name if it exist, or the original variable name. """ name_map = {} # Collect all the variables with moving average, including all # the trainable variables and variables which have been explicitly # added to the collection. moving_avg_variables = list( set(variables.moving_average_variables() + variables.trainable_variables())) for v in moving_avg_variables: name_map[self.average_name(v)] = v # Make sure we restore variables without moving average as well. for v in list( set(variables.all_variables()) - set(moving_avg_variables)): if v.op.name not in name_map: name_map[v.op.name] = v return name_map
def variables_to_restore(self): """Returns a map of names to `Variables` to restore. If a variable has a moving average, use the moving average variable name as the restore name; otherwise, use the variable name. For example, ```python variables_to_restore = ema.variables_to_restore() saver = tf.train.Saver(variables_to_restore) ``` Below is an example of such mapping: ``` conv/batchnorm/gamma/ExponentialMovingAverage: conv/batchnorm/gamma, conv_4/conv2d_params/ExponentialMovingAverage: conv_4/conv2d_params, global_step: global_step ``` Returns: A map from restore_names to variables. The restore_name can be the moving_average version of the variable name if it exist, or the original variable name. """ name_map = {} # Collect all the variables with moving average, including all # the trainable variables and variables which have been explicitly # added to the collection. moving_avg_variables = list(set(variables.moving_average_variables() + variables.trainable_variables())) for v in moving_avg_variables: name_map[self.average_name(v)] = v # Make sure we restore variables without moving average as well. for v in list(set(variables.all_variables()) - set(moving_avg_variables)): if v.op.name not in name_map: name_map[v.op.name] = v return name_map
def _CheckDecay(self, ema, actual_decay, dim): def _Scale(dk, steps): if ema._zero_debias: return 1 - dk**steps else: return 1 tens = _Repeat(10.0, dim) thirties = _Repeat(30.0, dim) var0 = variables.Variable(tens, name="v0") var1 = variables.Variable(thirties, name="v1") variables.global_variables_initializer().run() # Note that tensor2 is not a Variable but just a plain Tensor resulting # from the sum operation. tensor2 = var0 + var1 update = ema.apply([var0, var1, tensor2]) avg0 = ema.average(var0) avg1 = ema.average(var1) avg2 = ema.average(tensor2) self.assertItemsEqual([var0, var1], variables.moving_average_variables()) self.assertFalse(avg0 in variables.trainable_variables()) self.assertFalse(avg1 in variables.trainable_variables()) self.assertFalse(avg2 in variables.trainable_variables()) variables.global_variables_initializer().run() self.assertEqual("v0/ExponentialMovingAverage:0", avg0.name) self.assertEqual("v1/ExponentialMovingAverage:0", avg1.name) self.assertEqual("add/ExponentialMovingAverage:0", avg2.name) # Check initial values. self.assertAllClose(tens, self.evaluate(var0)) self.assertAllClose(thirties, self.evaluate(var1)) self.assertAllClose(_Repeat(10.0 + 30.0, dim), self.evaluate(tensor2)) # Check that averages are initialized correctly. self.assertAllClose(tens, self.evaluate(avg0)) self.assertAllClose(thirties, self.evaluate(avg1)) # Note that averages of Tensor's initialize to zeros_like since no value # of the Tensor is known because the Op has not been run (yet). self.assertAllClose(_Repeat(0.0, dim), self.evaluate(avg2)) # Update the averages and check. update.run() dk = actual_decay expected = _Repeat(10.0 * dk + 10.0 * (1 - dk), dim) self.assertAllClose(expected, self.evaluate(avg0)) expected = _Repeat(30.0 * dk + 30.0 * (1 - dk), dim) self.assertAllClose(expected, self.evaluate(avg1)) expected = _Repeat(0.0 * dk + (10.0 + 30.0) * (1 - dk) / _Scale(dk, 1), dim) self.assertAllClose(expected, self.evaluate(avg2)) # Again, update the averages and check. update.run() expected = _Repeat((10.0 * dk + 10.0 * (1 - dk)) * dk + 10.0 * (1 - dk), dim) self.assertAllClose(expected, self.evaluate(avg0)) expected = _Repeat((30.0 * dk + 30.0 * (1 - dk)) * dk + 30.0 * (1 - dk), dim) self.assertAllClose(expected, self.evaluate(avg1)) expected = _Repeat(((0.0 * dk + (10.0 + 30.0) * (1 - dk)) * dk + (10.0 + 30.0) * (1 - dk)) / _Scale(dk, 2), dim) self.assertAllClose(expected, self.evaluate(avg2))