def inference(self, inputs, n_classes): if (get_name_scope() != 'train') and (get_name_scope() != 'test'): raise Exception('name_scope is not train or test') reuse = self.get_resue() inputs = tf.identity(inputs, 'placeholder_inputs') op_ = self.op_set[get_name_scope()] with tf.variable_scope(self.model_name, reuse=reuse): tl.layers.set_name_reuse(reuse) network = self.network_func(inputs, get_name_scope() == 'train') op_['step'] = tf.get_variable(name='train_step', shape=(), dtype=tf.int32) op_['update_step'] = tf.assign(op_['step'], tf.add(op_['step'], tf.constant(1), name='train_step_add_one'), name='update_train_step') op_['network'] = DenseLayer(network, n_units=n_classes, act=tl.activation.identity, name='logits') op_['logits'] = op_['network'].outputs op_['softmax_logits'] = tf.nn.softmax(op_['logits']) op_['result'] = tf.argmax(op_['logits'], 1) return self
def optimizer(self, learning_rate): if (get_name_scope() != 'train') and (get_name_scope() != 'test'): raise Exception('name_scope is not train or test') op_ = self.op_set[get_name_scope()] optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate) op_['train_op'] = optimizer.minimize(op_['loss']) return self
def loss(self, labels): if (get_name_scope() != 'train') and (get_name_scope() != 'test'): raise Exception('name_scope is not train or test') op_ = self.op_set[get_name_scope()] op_['loss'] = tl.cost.cross_entropy(op_['logits'], labels, name='xentropy') return self
def summary_writer_function(name, tensor, function, family=None): """Helper function to write summaries. Args: name: name of the summary tensor: main tensor to form the summary function: function taking a tag and a scope which writes the summary family: optional, the summary's family Returns: The result of writing the summary. """ name_scope = ops.get_name_scope() if name_scope: # Add a slash to allow reentering the name scope. name_scope += "/" def record(): with ops.name_scope(name_scope), summary_op_util.summary_scope( name, family, values=[tensor]) as (tag, scope): with ops.control_dependencies([function(tag, scope)]): return constant_op.constant(True) if context.context().summary_writer_resource is None: return control_flow_ops.no_op() with ops.device("cpu:0"): op = utils.smart_cond( should_record_summaries(), record, _nothing, name="") ops.add_to_collection(ops.GraphKeys._SUMMARY_COLLECTION, op) # pylint: disable=protected-access return op
def _var_to_tensor(var, dtype=None, name=None, as_ref=False): """Converts a `ShardedVariable` to a `Tensor`.""" del name if dtype is not None and not dtype.is_compatible_with(var.dtype): raise ValueError( 'Incompatible type conversion requested to type {!r} for variable ' 'of type {!r}'.format(dtype.name, var.dtype.name)) if as_ref: raise NotImplementedError( "ShardedVariable doesn't support being used as a reference.") # We use op dispatch mechanism to override embedding_lookup ops when called # with ShardedVariable. This requires embedding_lookup ops to raise TypeError # when called with ShardedVariable. However since ShardedVariable can be # converted to a tensor via concat, embedding_lookup ops would silently # do the convertion and never raise a TypeError. To be able to properly # raise a TypeError, namescope is used to detect if this method is called # within a embedding_lookup op. # NOTE: This doesn't work in eager mode since op namescope is always cleared # in eager. This also breaks if user sets the name of embedding_lookup op # with something that doesn't contain str "embedding_lookup". # # TODO(chenkai): Find a more robust way to do this, which should not rely # on namescope. if 'embedding_lookup' in ops.get_name_scope(): raise TypeError( 'Converting ShardedVariable to tensor in embedding lookup' ' ops is disallowed.') return array_ops.concat(var.variables, axis=0)
def summary_writer_function(name, tensor, function, family=None): """Helper function to write summaries. Args: name: name of the summary tensor: main tensor to form the summary function: function taking a tag and a scope which writes the summary family: optional, the summary's family Returns: The result of writing the summary. """ name_scope = ops.get_name_scope() if name_scope: # Add a slash to allow reentering the name scope. name_scope += "/" def record(): with ops.name_scope(name_scope), summary_op_util.summary_scope( name, family, values=[tensor]) as (tag, scope): with ops.control_dependencies([function(tag, scope)]): return constant_op.constant(True) if context.context().summary_writer_resource is None: return control_flow_ops.no_op() with ops.device("cpu:0"): op = smart_cond.smart_cond( should_record_summaries(), record, _nothing, name="") if not context.executing_eagerly(): ops.add_to_collection(ops.GraphKeys._SUMMARY_COLLECTION, op) # pylint: disable=protected-access return op
def Body(i): with ops.name_scope("body"): actual_name_scope = ops.get_name_scope() expected_name_scope = "foo/while/body" assert actual_name_scope == expected_name_scope, ( "%s does not match %s" % (actual_name_scope, expected_name_scope)) return i
def Cond(unused_i): with ops.name_scope("cond"): actual_name_scope = ops.get_name_scope() expected_name_scope = "foo/while/cond" assert actual_name_scope == expected_name_scope, ( "%s does not match %s" % (actual_name_scope, expected_name_scope)) return False
def accuracy(self, labels, n_classes): if (get_name_scope() != 'train') and (get_name_scope() != 'test'): raise Exception('name_scope is not train or test') op_ = self.op_set[get_name_scope()] op_['total_accuracy'] = tf.reduce_mean( tf.cast(tf.equal(tf.argmax(op_['logits'], 1), labels), tf.float32)) op_['class_accuracy'] = list() for c in range(n_classes): tindex = tf.reshape(tf.where(tf.equal(labels, c)), [-1]) tlables = tf.nn.embedding_lookup(labels, tindex) toutput = tf.nn.embedding_lookup(op_['logits'], tindex) tacc = tf.reduce_mean( tf.cast(tf.equal(tf.argmax(toutput, 1), tlables), tf.float32)) op_['class_accuracy'].append(tacc) return self
def update_fn(v, value, biased_var, local_step): update_biased = state_ops.assign_sub(biased_var, (biased_var - value) * decay) update_local_step = local_step.assign_add(1) # This function gets `1 - decay`, so use `1.0 - decay` in the exponent. bias_factor = 1 - math_ops.pow(1.0 - decay, update_local_step) return state_ops.assign( v, update_biased / bias_factor, name=ops.get_name_scope() + "/")
def update_fn(v, value, biased_var, local_step): update_biased = state_ops.assign_sub(biased_var, (biased_var - value) * decay) update_local_step = local_step.assign_add(1) # This function gets `1 - decay`, so use `1.0 - decay` in the exponent. bias_factor = 1 - math_ops.pow(1.0 - decay, update_local_step) return state_ops.assign( v, update_biased / bias_factor, name=ops.get_name_scope() + "/")
def _gen_unique_name(self, var_name): name_scope = ops.get_name_scope() if name_scope is None or "" == name_scope: return ops.get_default_graph().unique_name(var_name, mark_as_used=True) else: # TODO: use regex while name_scope[-1] == r"/": name_scope = name_scope[:-1] return name_scope
def record_gradient(op_name, inputs, attrs, outputs): """Explicitly record the gradient for a given op. Args: op_name: The op name as listed in the `OpDef` for the op. inputs: A list of tensor inputs to the op. attrs: The op attributes as a flattened list of alternating attribute names and attribute values. outputs: A list of tensor outputs from the op. """ pywrap_tfe.TFE_Py_RecordGradient(op_name, inputs, attrs, outputs, ops.get_name_scope())
def auto_naming(self, name=None): ''' Create unique name for each module automatically ''' # create name dictionary if not hasattr(auto_naming, 'name_dict'): setattr(auto_naming, 'name_dict', {}) def to_snake_case(name): intermediate = re.sub('(.)([A-Z][a-z0-9]+)', r'\1_\2', name) insecure = re.sub('([a-z])([A-Z])', r'\1_\2', intermediate).lower() if insecure[0] != '_': return insecure return 'private' + insecure # get default name if name is None: name = to_snake_case(self.__class__.__name__) # create key key = name # get current name scope. # I don't know why tf.compat.v1.get_default_graph().get_name_scope() doesn't work, # so I use tensorflow.python.framework.ops.get_name_scope(), instead. This function # is not officially documented. (WARNING) name_scope = ops.get_name_scope() if name_scope: key = os.path.join(name_scope, key) # name does not exist if key not in auto_naming.name_dict: auto_naming.name_dict[key] = 0 # name already exists else: # increate counter auto_naming.name_dict[key] += 1 name = '{}_{}'.format(name, auto_naming.name_dict[key]) return name
def summary_scope(name, default_name="summary", values=None): """Experimental context manager for use when defining a custom summary op. This behaves similarly to `tf.name_scope`, except that it returns a generated summary tag in addition to the scope name. The tag is structurally similar to the scope name - derived from the user-provided name, prefixed with enclosing name scopes if any - but we relax the constraint that it be uniquified, as well as the character set limitation (so the user-provided name can contain characters not legal for scope names; in the scope name these are removed). This makes the summary tag more predictable and consistent for the user. For example, to define a new summary op called `my_op`: ```python def my_op(name, my_value, step): with tf.summary.summary_scope(name, "MyOp", [my_value]) as (tag, scope): my_value = tf.convert_to_tensor(my_value) return tf.summary.write(tag, my_value, step=step) ``` Args: name: string name for the summary. default_name: Optional; if provided, used as default name of the summary. values: Optional; passed as `values` parameter to name_scope. Yields: A tuple `(tag, scope)` as described above. """ name = name or default_name current_scope = ops.get_name_scope() tag = current_scope + "/" + name if current_scope else name # Strip illegal characters from the scope name, and if that leaves nothing, # use None instead so we pick up the default name. name = _INVALID_SCOPE_CHARACTERS.sub("", name) or None with ops.name_scope(name, default_name, values, skip_on_eager=False) as scope: yield tag, scope
def summary_scope(name, default_name="summary", values=None): """Experimental context manager for use when defining a custom summary op. This behaves similarly to `tf.name_scope`, except that it returns a generated summary tag in addition to the scope name. The tag is structurally similar to the scope name - derived from the user-provided name, prefixed with enclosing name scopes if any - but we relax the constraint that it be uniquified, as well as the character set limitation (so the user-provided name can contain characters not legal for scope names; in the scope name these are removed). This makes the summary tag more predictable and consistent for the user. For example, to define a new summary op called `my_op`: ```python def my_op(name, my_value, step): with tf.summary.summary_scope(name, "MyOp", [my_value]) as (tag, scope): my_value = tf.convert_to_tensor(my_value) return tf.summary.write(tag, my_value, step=step) ``` Args: name: string name for the summary. default_name: Optional; if provided, used as default name of the summary. values: Optional; passed as `values` parameter to name_scope. Yields: A tuple `(tag, scope)` as described above. """ name = name or default_name current_scope = ops.get_name_scope() tag = current_scope + "/" + name if current_scope else name # Strip illegal characters from the scope name, and if that leaves nothing, # use None instead so we pick up the default name. name = _INVALID_SCOPE_CHARACTERS.sub("", name) or None with ops.name_scope(name, default_name, values) as scope: yield tag, scope
def _InsertQuantOp(context, name, producer, consumers, is_training, moving_avg=True, init_min=-6.0, init_max=6.0, bits=8, ema_decay=0.999, quant_delay=None, vars_collection=ops.GraphKeys.GLOBAL_VARIABLES, narrow_range=False, producer_scope=None, consumer_scope=None): """Inserts a quant op between a producer op and (multiple) consumer ops. Args: context: Context where producer and consumer operations are nested. name: Name for the new quantization op within the context. producer: Producer operation of the pairs where quantization will be inserted. consumers: Consumer operations of the pairs. is_training: Whether quantizing training graph or eval graph. moving_avg: Specifies whether to use exponential moving average or just the last value seen. init_min: Starting minimum value for the new quantization op. init_max: Starting maximum value for the new quantization op. bits: Number of bits to use for quantization, must be between 2 and 8. ema_decay: (Optional) Float, EMA decay parameter. EMA is used to update quantization intervals for quantizing activations (see here about EMA: https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average). quant_delay: (Optional, default None) Int, count of global steps for which to delay quantization. This helps weights stabilize at the start of training. vars_collection: (Optional) Collection where to store the variables for quantization interval ends. narrow_range: Whether to use the narrow quantization range [1; 2^bits - 1] or wide range [0; 2^bits - 1]. producer_scope: The restriction of producer scope. If not None, the new op will be inserted only when the producer is in this scope. consumer_scope: The restriction of producer scope. If not None, the new op will be inserted only when all the consumers are in this scope. Raises: ValueError: When producer operation is not directly connected to the consumer operation. """ if producer_scope and not producer.name.startswith(producer_scope): logging.info( '_InsertQuantOp ignores context="%s" name="%s" ' 'because producer "%s" is not in scope "%s"', context, name, producer.name, producer_scope) return if consumer_scope: consumers_in_scope = [] for consumer in consumers: if consumer.name.startswith(consumer_scope): consumers_in_scope.append(consumer) else: logging.info( '_InsertQuantOp context="%s" name="%s" ignores ' 'consumer "%s" because it is not in scope "%s"', context, name, consumer.name, consumer_scope) return consumers = consumers_in_scope name_prefix = _AddContextToName(context, name) # This is needed on TPU where name_scope == 'TPUReplicate/loop', and # name_prefix starts with 'TPUReplicate/loop/'; without dropping it # variables are created as TPUReplicate/loop/TPUReplicate/loop/..., which # breaks things later. name_scope = ops.get_name_scope() if name_scope: name_prefix = common.DropStringPrefix(name_prefix, name_scope + '/') inputs = producer.outputs[0] # Prevent ops from being quantized multiple times. Bypass ops can sometimes # overlap between multiple matches, so we need to ensure that we don't # add duplicate FakeQuant operations. fake_quant_ops = set([ 'FakeQuantWithMinMaxVars', 'FakeQuantWithMinMaxArgs' ]) if fake_quant_ops.intersection(set([c.type for c in inputs.consumers()])): return if moving_avg: quant = ( quant_ops.MovingAvgQuantize( inputs, init_min=init_min, init_max=init_max, ema_decay=ema_decay, is_training=is_training, num_bits=bits, narrow_range=narrow_range, vars_collection=vars_collection, name_prefix=name_prefix)) else: quant = ( quant_ops.LastValueQuantize( inputs, init_min=init_min, init_max=init_max, is_training=is_training, num_bits=bits, narrow_range=narrow_range, vars_collection=vars_collection, name_prefix=name_prefix)) if quant_delay and quant_delay > 0: activate_quant = math_ops.greater_equal( common.CreateOrGetQuantizationStep(), quant_delay, name=name_prefix + '/activate_quant') quant = control_flow_ops.cond( activate_quant, lambda: quant, lambda: inputs, name=name_prefix + '/delayed_quant') if consumers: tensors_modified_count = graph_editor.reroute_ts( [quant], [inputs], can_modify=consumers) # Some operations can have multiple output tensors going to the same # consumer. Since consumers is a set, we need to ensure that # tensors_modified_count is greater than or equal to the length of the set # of consumers. if tensors_modified_count < len(consumers): raise ValueError('No inputs quantized for ops: [%s]' % ', '.join( [consumer.name for consumer in consumers]))
def call(self, inputs): self.active_name_scope = ops.get_name_scope() return inputs
def _InsertQuantOp(context, name, producer, consumers, is_training, moving_avg=True, init_min=-6.0, init_max=6.0, bits=8, ema_decay=0.999, quant_delay=None, vars_collection=ops.GraphKeys.GLOBAL_VARIABLES, narrow_range=False): """Inserts a quant op between a producer op and (multiple) consumer ops. Args: context: Context where producer and consumer operations are nested. name: Name for the new quantization op within the context. producer: Producer operation of the pairs where quantization will be inserted. consumers: Consumer operations of the pairs. is_training: Whether quantizing training graph or eval graph. moving_avg: Specifies whether to use exponential moving average or just the last value seen. init_min: Starting minimum value for the new quantization op. init_max: Starting maximum value for the new quantization op. bits: Number of bits to use for quantization, must be between 2 and 8. ema_decay: (Optional) Float, EMA decay parameter. EMA is used to update quantization intervals for quantizing activations (see here about EMA: https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average). quant_delay: (Optional, default None) Int, count of global steps for which to delay quantization. This helps weights stabilize at the start of training. vars_collection: (Optional) Collection where to store the variables for quantization interval ends. narrow_range: Whether to use the narrow quantization range [1; 2^bits - 1] or wide range [0; 2^bits - 1]. Raises: ValueError: When producer operation is not directly connected to the consumer operation. """ name_prefix = _AddContextToName(context, name) # This is needed on TPU where name_scope == 'TPUReplicate/loop', and # name_prefix starts with 'TPUReplicate/loop/'; without dropping it # variables are created as TPUReplicate/loop/TPUReplicate/loop/..., which # breaks things later. name_prefix = common.DropStringPrefix(name_prefix, ops.get_name_scope() + '/') inputs = producer.outputs[0] if moving_avg: quant = (quant_ops.MovingAvgQuantize(inputs, init_min=init_min, init_max=init_max, ema_decay=ema_decay, is_training=is_training, num_bits=bits, narrow_range=narrow_range, vars_collection=vars_collection, name_prefix=name_prefix)) else: quant = (quant_ops.LastValueQuantize(inputs, init_min=init_min, init_max=init_max, is_training=is_training, num_bits=bits, narrow_range=narrow_range, vars_collection=vars_collection, name_prefix=name_prefix)) if quant_delay and quant_delay > 0: activate_quant = math_ops.greater_equal( common.CreateOrGetQuantizationStep(), quant_delay, name=name_prefix + '/activate_quant') quant = control_flow_ops.cond(activate_quant, lambda: quant, lambda: inputs, name=name_prefix + '/delayed_quant') nodes_modified_count = graph_editor.reroute_ts([quant], [inputs], can_modify=consumers) if nodes_modified_count != len(consumers): raise ValueError('Some inputs not quantized for ops: [%s]' % ', '.join([consumer.name for consumer in consumers]))
def _InsertQuantOp(context, name, producer, consumers, is_training, moving_avg=True, init_min=-6.0, init_max=6.0, bits=8, ema_decay=0.999, quant_delay=None, vars_collection=ops.GraphKeys.GLOBAL_VARIABLES, narrow_range=False, producer_scope=None, consumer_scope=None): """Inserts a quant op between a producer op and (multiple) consumer ops. Args: context: Context where producer and consumer operations are nested. name: Name for the new quantization op within the context. producer: Producer operation of the pairs where quantization will be inserted. consumers: Consumer operations of the pairs. is_training: Whether quantizing training graph or eval graph. moving_avg: Specifies whether to use exponential moving average or just the last value seen. init_min: Starting minimum value for the new quantization op. init_max: Starting maximum value for the new quantization op. bits: Number of bits to use for quantization, must be between 2 and 8. ema_decay: (Optional) Float, EMA decay parameter. EMA is used to update quantization intervals for quantizing activations (see here about EMA: https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average). quant_delay: (Optional, default None) Int, count of global steps for which to delay quantization. This helps weights stabilize at the start of training. vars_collection: (Optional) Collection where to store the variables for quantization interval ends. narrow_range: Whether to use the narrow quantization range [1; 2^bits - 1] or wide range [0; 2^bits - 1]. producer_scope: The restriction of producer scope. If not None, the new op will be inserted only when the producer is in this scope. consumer_scope: The restriction of producer scope. If not None, the new op will be inserted only when all the consumers are in this scope. Raises: ValueError: When producer operation is not directly connected to the consumer operation. """ if producer_scope and not producer.name.startswith(producer_scope): logging.info( '_InsertQuantOp ignores context="%s" name="%s" ' 'because producer "%s" is not in scope "%s"', context, name, producer.name, producer_scope) return if consumer_scope: consumers_in_scope = [] for consumer in consumers: if consumer.name.startswith(consumer_scope): consumers_in_scope.append(consumer) else: logging.info( '_InsertQuantOp context="%s" name="%s" ignores ' 'consumer "%s" because it is not in scope "%s"', context, name, consumer.name, consumer_scope) return consumers = consumers_in_scope name_prefix = _AddContextToName(context, name) # This is needed on TPU where name_scope == 'TPUReplicate/loop', and # name_prefix starts with 'TPUReplicate/loop/'; without dropping it # variables are created as TPUReplicate/loop/TPUReplicate/loop/..., which # breaks things later. name_scope = ops.get_name_scope() if name_scope: name_prefix = common.DropStringPrefix(name_prefix, name_scope + '/') inputs = producer.outputs[0] # Prevent ops from being quantized multiple times. Bypass ops can sometimes # overlap between multiple matches, so we need to ensure that we don't # add duplicate FakeQuant operations. fake_quant_ops = set( ['FakeQuantWithMinMaxVars', 'FakeQuantWithMinMaxArgs']) if fake_quant_ops.intersection(set([c.type for c in inputs.consumers()])): return if moving_avg: quant = (quant_ops.MovingAvgQuantize(inputs, init_min=init_min, init_max=init_max, ema_decay=ema_decay, is_training=is_training, num_bits=bits, narrow_range=narrow_range, vars_collection=vars_collection, name_prefix=name_prefix)) else: quant = (quant_ops.LastValueQuantize(inputs, init_min=init_min, init_max=init_max, is_training=is_training, num_bits=bits, narrow_range=narrow_range, vars_collection=vars_collection, name_prefix=name_prefix)) if quant_delay and quant_delay > 0: activate_quant = math_ops.greater_equal( common.CreateOrGetQuantizationStep(), quant_delay, name=name_prefix + '/activate_quant') quant = control_flow_ops.cond(activate_quant, lambda: quant, lambda: inputs, name=name_prefix + '/delayed_quant') if consumers: tensors_modified_count = graph_editor.reroute_ts([quant], [inputs], can_modify=consumers) # Some operations can have multiple output tensors going to the same # consumer. Since consumers is a set, we need to ensure that # tensors_modified_count is greater than or equal to the length of the set # of consumers. if tensors_modified_count < len(consumers): raise ValueError( 'No inputs quantized for ops: [%s]' % ', '.join([consumer.name for consumer in consumers]))
def _record_gradient(op_name, inputs, attrs, results): return pywrap_tfe.TFE_Py_RecordGradient(op_name, inputs, attrs, results, ops.get_name_scope())
def _get_previous_name_scope(): current_name_scope = ops.get_name_scope() return current_name_scope.rsplit('/', 1)[0] + '/'
def _InsertCalibOp(context, name, producer, consumers, vars_collection=ops.GraphKeys.GLOBAL_VARIABLES, producer_scope=None, consumer_scope=None): """Inserts calibration ops between a producer op and (multiple) consumer ops. Args: context: Context where producer and consumer operations are nested. name: Name for the new calibration op within the context. producer: Producer operation of the pairs where calibration will be inserted. consumers: Consumer operations of the pairs. producer_scope: The restriction of producer scope. If not None, the new op will be inserted only when the producer is in this scope. consumer_scope: The restriction of consumer scope. If not None, the new op will be inserted only when all the consumers are in this scope. Raises: ValueError: When producer operation is not directly connected to the consumer operation. """ if producer_scope and not producer.name.startswith(producer_scope): logging.info( '_InsertCalibOp ignores context="%s" name="%s" ' 'because producer "%s" is not in scope "%s"', context, name, producer.name, producer_scope) return if consumer_scope: consumers_in_scope = [] for consumer in consumers: if consumer.name.startswith(consumer_scope): consumers_in_scope.append(consumer) else: logging.info( '_InsertCalibOp context="%s" name="%s" ignores ' 'consumer "%s" because it is not in scope "%s"', context, name, consumer.name, consumer_scope) return consumers = consumers_in_scope name_prefix = _AddContextToName(context, name) name_scope = ops.get_name_scope() if name_scope: name_prefix = common.DropStringPrefix(name_prefix, name_scope + '/') inputs = producer.outputs[0] # Prevent ops from being modified multiple times. Bypass ops can sometimes # overlap between multiple matches, so we need to ensure that we don't # add duplicate calibration operations. #if _FollowedByFakeQuant(inputs): # return with variable_scope.variable_scope(None, default_name=name_prefix, values=[inputs]) as scope: # Currently no per channel. min_max_shape = [] vars_collections = [vars_collection] if vars_collection else [] min_var = _ModelVariable('min', shape=min_max_shape, initializer=init_ops.constant_initializer( float('inf')), collections=vars_collections, trainable=False) max_var = _ModelVariable( 'max', shape=min_max_shape, initializer=init_ops.constant_initializer(-float('inf')), collections=vars_collections, trainable=False) batch_min = math_ops.reduce_min(inputs, name='BatchMin') batch_max = math_ops.reduce_max(inputs, name='BatchMax') range_min = math_ops.minimum(batch_min, min_var, name=name_prefix + '/range_min') range_max = math_ops.maximum(batch_max, max_var, name=name_prefix + '/range_max') return range_min, range_max
def _InsertQuantOp(context, name, producer, consumers, is_training, moving_avg=True, init_min=-6.0, init_max=6.0, bits=8, ema_decay=0.999, quant_delay=None, vars_collection=ops.GraphKeys.GLOBAL_VARIABLES, narrow_range=False): """Inserts a quant op between a producer op and (multiple) consumer ops. Args: context: Context where producer and consumer operations are nested. name: Name for the new quantization op within the context. producer: Producer operation of the pairs where quantization will be inserted. consumers: Consumer operations of the pairs. is_training: Whether quantizing training graph or eval graph. moving_avg: Specifies whether to use exponential moving average or just the last value seen. init_min: Starting minimum value for the new quantization op. init_max: Starting maximum value for the new quantization op. bits: Number of bits to use for quantization, must be between 2 and 8. ema_decay: (Optional) Float, EMA decay parameter. EMA is used to update quantization intervals for quantizing activations (see here about EMA: https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average). quant_delay: (Optional, default None) Int, count of global steps for which to delay quantization. This helps weights stabilize at the start of training. vars_collection: (Optional) Collection where to store the variables for quantization interval ends. narrow_range: Whether to use the narrow quantization range [1; 2^bits - 1] or wide range [0; 2^bits - 1]. Raises: ValueError: When producer operation is not directly connected to the consumer operation. """ name_prefix = _AddContextToName(context, name) # This is needed on TPU where name_scope == 'TPUReplicate/loop', and # name_prefix starts with 'TPUReplicate/loop/'; without dropping it # variables are created as TPUReplicate/loop/TPUReplicate/loop/..., which # breaks things later. name_prefix = common.DropStringPrefix(name_prefix, ops.get_name_scope() + '/') inputs = producer.outputs[0] if moving_avg: quant = ( quant_ops.MovingAvgQuantize( inputs, init_min=init_min, init_max=init_max, ema_decay=ema_decay, is_training=is_training, num_bits=bits, narrow_range=narrow_range, vars_collection=vars_collection, name_prefix=name_prefix)) else: quant = ( quant_ops.LastValueQuantize( inputs, init_min=init_min, init_max=init_max, is_training=is_training, num_bits=bits, narrow_range=narrow_range, vars_collection=vars_collection, name_prefix=name_prefix)) if quant_delay and quant_delay > 0: activate_quant = math_ops.greater_equal( common.CreateOrGetQuantizationStep(), quant_delay, name=name_prefix + '/activate_quant') quant = control_flow_ops.cond( activate_quant, lambda: quant, lambda: inputs, name=name_prefix + '/delayed_quant') nodes_modified_count = graph_editor.reroute_ts( [quant], [inputs], can_modify=consumers) if nodes_modified_count != len(consumers): raise ValueError('Some inputs not quantized for ops: [%s]' % ', '.join( [consumer.name for consumer in consumers]))
def _InsertQuantOp(context, name, producer, consumers, is_training, moving_avg=True, init_min=-6.0, init_max=6.0, bits=8, symmetric=False, ema_decay=0.999, quant_delay=None, vars_collection=ops.GraphKeys.GLOBAL_VARIABLES, narrow_range=False, producer_scope=None, consumer_scope=None): """Inserts a quant op between a producer op and (multiple) consumer ops. Args: context: Context where producer and consumer operations are nested. name: Name for the new quantization op within the context. producer: Producer operation of the pairs where quantization will be inserted. consumers: Consumer operations of the pairs. is_training: Whether quantizing training graph or eval graph. moving_avg: Specifies whether to use exponential moving average or just the last value seen. init_min: Starting minimum value for the new quantization op. init_max: Starting maximum value for the new quantization op. bits: Number of bits to use for quantization, must be between 2 and 8. symmetric: (Optional) If true, use symmetric quantization limits instead of training the minimum and maximum of each quantization range separately. ema_decay: (Optional) Float, EMA decay parameter. EMA is used to update quantization intervals for quantizing activations (see here about EMA: https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average). quant_delay: (Optional, default None) Int, count of global steps for which to delay quantization. This helps weights stabilize at the start of training. vars_collection: (Optional) Collection where to store the variables for quantization interval ends. narrow_range: Whether to use the narrow quantization range [1; 2^bits - 1] or wide range [0; 2^bits - 1]. producer_scope: The restriction of producer scope. If not None, the new op will be inserted only when the producer is in this scope. consumer_scope: The restriction of producer scope. If not None, the new op will be inserted only when all the consumers are in this scope. Raises: ValueError: When producer operation is not directly connected to the consumer operation. """ if producer_scope and not producer.name.startswith(producer_scope): logging.info( '_InsertQuantOp ignores context="%s" name="%s" ' 'because producer "%s" is not in scope "%s"', context, name, producer.name, producer_scope) return if consumer_scope: consumers_in_scope = [] for consumer in consumers: if consumer.name.startswith(consumer_scope): consumers_in_scope.append(consumer) else: logging.info( '_InsertQuantOp context="%s" name="%s" ignores ' 'consumer "%s" because it is not in scope "%s"', context, name, consumer.name, consumer_scope) return consumers = consumers_in_scope name_prefix = _AddContextToName(context, name) # This is needed on TPU where name_scope == 'TPUReplicate/loop', and # name_prefix starts with 'TPUReplicate/loop/'; without dropping it # variables are created as TPUReplicate/loop/TPUReplicate/loop/..., which # breaks things later. name_scope = ops.get_name_scope() if name_scope: name_prefix = common.DropStringPrefix(name_prefix, name_scope + '/') inputs = producer.outputs[0] # Prevent ops from being quantized multiple times. Bypass ops can sometimes # overlap between multiple matches, so we need to ensure that we don't # add duplicate FakeQuant operations. fake_quant_op = _GetFollowingFakeQuantOp(inputs) # If we find that we are attempting to insert a fake quant op following # a fake quant, we skip inserting a fake quant op if fake_quant_op is None: if moving_avg: quant = (quant_ops.MovingAvgQuantize( inputs, init_min=init_min, init_max=init_max, ema_decay=ema_decay, is_training=is_training, num_bits=bits, symmetric=symmetric, narrow_range=narrow_range, vars_collection=vars_collection, name_prefix=name_prefix)) else: quant = (quant_ops.LastValueQuantize( inputs, init_min=init_min, init_max=init_max, is_training=is_training, num_bits=bits, symmetric=symmetric, narrow_range=narrow_range, vars_collection=vars_collection, name_prefix=name_prefix)) if quant_delay and quant_delay > 0: activate_quant = math_ops.greater_equal( common.CreateOrGetQuantizationStep(), quant_delay, name=name_prefix + '/activate_quant') quant = control_flow_ops.cond(activate_quant, lambda: quant, lambda: inputs, name=name_prefix + '/delayed_quant') else: # return # If a fake quant op is present already, make sure that # any downstream use of the tensor reroutes to the appropriate quantized # tensor. If there is no quant_delay, this is simply the output of the # fake quant op. If there is a quant delay, we reroute to the output # of the delayed quant operation, which inserts quantization only after # a specified quant_delay quant = fake_quant_op.outputs[0] if quant_delay and quant_delay > 0: name_prefix = '/'.join(quant.name.split('/')[:-1]) quant = quant.graph.get_tensor_by_name(name_prefix + '/delayed_quant/Merge:0') pruned_consumer_set = set() for consumer in consumers: fake_quant_dest_op = _GetFollowingFakeQuantOp(consumer.outputs[0]) if (fake_quant_dest_op is None or fake_quant_dest_op.name != fake_quant_op.name): pruned_consumer_set.add(consumer) consumers = pruned_consumer_set # If we have # input->pass_through->fake_quant # there is nothing to reroute. # # If we have # input-> pass_through->fake_quant # |-> consumer # Then we reroute such that: # input-> pass_through->fake_quant # |-> consumer if consumers: tensors_modified_count = common.RerouteTensor(quant, inputs, can_modify=consumers) # Some operations can have multiple output tensors going to the same # consumer. Since consumers is a set, we need to ensure that # tensors_modified_count is greater than or equal to the length of the set # of consumers. if tensors_modified_count < len(consumers): raise ValueError( 'No inputs quantized for ops: [%s]' % ', '.join([consumer.name for consumer in consumers]))
def _get_previous_name_scope(): current_name_scope = ops.get_name_scope() return current_name_scope.rsplit('/', 1)[0] + '/'