def __init__(self,
output_boundary: List[tf.Operation],
threshold,
l1_fraction=0.0,
regularizer_decorator: Optional[Type[
generic_regularizers.OpRegularizer]] = None,
decorator_parameters=None,
input_boundary: Optional[List[tf.Operation]] = None,
force_group: Optional[List[Text]] = None,
regularizer_blacklist: Optional[List[Text]] = None):
"""Creates a GroupLassoModelSizeRegularizer object.
Args:
output_boundary: An OpRegularizer will be created for all these
operations, and recursively for all ops they depend on via data
dependency that does not involve ops from input_boundary.
threshold: A float scalar, will be used as a 'threshold' for all
regularizer instances created by this class.
l1_fraction: A float scalar. The relative weight of L1 in L1 + L2
regularization.
regularizer_decorator: A class of OpRegularizer decorator to use.
decorator_parameters: A dictionary of parameters to pass to the decorator
factory. To be used only with decorators that requires parameters,
otherwise use None.
input_boundary: A list of ops that represent the input boundary of the
subgraph being regularized (input boundary is not regularized).
force_group: List of regex for ops that should be force-grouped. Each
regex corresponds to a separate group. Use '|' operator to specify
multiple patterns in a single regex. See op_regularizer_manager for more
detail.
regularizer_blacklist: List of regex for ops that should not be
regularized. See op_regularizer_manager for more detail.
"""
custom_handlers = {
'Conv2D':
conv_handler.ConvSourceOpHandler(threshold, l1_fraction),
'Conv3D':
conv_handler.ConvSourceOpHandler(threshold, l1_fraction),
'Conv2DBackpropInput':
conv2d_transpose_handler.Conv2DTransposeSourceOpHandler(
threshold, l1_fraction),
'MatMul':
matmul_handler.MatMulSourceOpHandler(threshold, l1_fraction)
}
if regularizer_decorator:
for key in custom_handlers:
custom_handlers[key] = op_handler_decorator.OpHandlerDecorator(
custom_handlers[key], regularizer_decorator, decorator_parameters)
op_handler_dict = op_handlers.get_group_lasso_op_handler_dict()
op_handler_dict.update(custom_handlers)
self._manager = orm.OpRegularizerManager(
output_boundary,
op_handler_dict,
input_boundary=input_boundary,
force_group=force_group,
regularizer_blacklist=regularizer_blacklist)
self._calculator = cost_calculator.CostCalculator(
self._manager, resource_function.model_size_function)
def __init__(
self,
ops,
threshold,
l1_fraction=0,
regularizer_decorator: Type[generic_regularizers.OpRegularizer] = None,
decorator_parameters=None,
force_group=None,
regularizer_blacklist=None):
"""Creates a GroupLassoActivationRegularizer object.
Args:
ops: A list of tf.Operation. An OpRegularizer will be created for all the
ops in `ops`, and recursively for all ops they depend on via data
dependency. Typically `ops` would contain a single tf.Operation, which
is the output of the network.
threshold: A float scalar, will be used as a 'threshold' for all
regularizer instances created by this class.
l1_fraction: Relative weight of L1 in L1 + L2 regularization.
regularizer_decorator: A class of OpRegularizer decorator to use.
decorator_parameters: A dictionary of parameters to pass to the decorator
factory. To be used only with decorators that requires parameters,
otherwise use None.
force_group: List of regex for ops that should be force-grouped. Each
regex corresponds to a separate group. Use '|' operator to specify
multiple patterns in a single regex. See op_regularizer_manager for more
detail.
regularizer_blacklist: List of regex for ops that should not be
regularized. See op_regularizer_manager for more detail.
"""
conv2d_handler = conv2d_source_op_handler.Conv2DSourceOpHandler(
threshold, l1_fraction)
conv2d_transpose_handler = (
conv2d_transpose_source_op_handler.Conv2DTransposeSourceOpHandler(
threshold, l1_fraction))
matmul_handler = matmul_source_op_handler.MatMulSourceOpHandler(
threshold, l1_fraction)
if regularizer_decorator:
conv2d_handler = op_handler_decorator.OpHandlerDecorator(
conv2d_handler, regularizer_decorator, decorator_parameters)
conv2d_transpose_handler = op_handler_decorator.OpHandlerDecorator(
conv2d_transpose_handler, regularizer_decorator, decorator_parameters)
matmul_handler = op_handler_decorator.OpHandlerDecorator(
matmul_handler, regularizer_decorator, decorator_parameters)
op_handler_dict = op_handlers.get_group_lasso_op_handler_dict()
op_handler_dict.update({
'Conv2D': conv2d_handler,
'Conv2DBackpropInput': conv2d_transpose_handler,
'MatMul': matmul_handler,
})
self._manager = orm.OpRegularizerManager(
ops,
op_handler_dict,
force_group=force_group,
regularizer_blacklist=regularizer_blacklist)
self._calculator = cost_calculator.CostCalculator(
self._manager, resource_function.activation_count_function)
def testMatMul2D(self, size):
inputs = tf.zeros((13, 2))
handler = matmul_source_op_handler.MatMulSourceOpHandler(0.1)
kernel = tf.constant([[1, 2, 3], [4, 5, 6]], dtype=tf.float32)
x = tf.matmul(inputs, kernel, transpose_b=False, name='MatMul')
op_slice = orm.OpSlice(x.op, orm.Slice(0, size))
transpose_kernel = tf.constant([[1, 4], [2, 5], [3, 6]], dtype=tf.float32)
x_other = tf.matmul(
inputs,
transpose_kernel,
transpose_b=True,
name='MatMulTransposedKernel')
op_slice_other = orm.OpSlice(x_other.op, orm.Slice(0, size))
self.assertAllClose(
handler.create_regularizer(op_slice).regularization_vector,
handler.create_regularizer(op_slice_other).regularization_vector)
def __init__(self,
output_boundary,
threshold,
hardware,
batch_size=1,
l1_fraction=0,
regularizer_decorator=None,
decorator_parameters=None,
input_boundary=None,
force_group=None,
regularizer_blacklist=None,
convert_to_variable=True):
"""Creates a GroupLassoFlopsRegularizer object.
Args:
output_boundary: An OpRegularizer will be created for all these
operations, and recursively for all ops they depend on via data
dependency that does not involve ops from input_boundary.
threshold: A float scalar, will be used as a 'threshold' for all
regularizer instances created by this class.
hardware: String name of hardware platform to target. Must be a key from
resource_function.PEAK_COMPUTE.
batch_size: Integer batch size to calculate cost/loss for.
l1_fraction: Relative weight of L1 in L1 + L2 regularization.
regularizer_decorator: A class of OpRegularizer decorator to use.
decorator_parameters: A dictionary of parameters to pass to the decorator
factory. To be used only with decorators that requires parameters,
otherwise use None.
input_boundary: A list of ops that represent the input boundary of the
subgraph being regularized (input boundary is not regularized).
force_group: List of regex for ops that should be force-grouped. Each
regex corresponds to a separate group. Use '|' operator to specify
multiple patterns in a single regex. See op_regularizer_manager for more
detail.
regularizer_blacklist: List of regex for ops that should not be
regularized. See op_regularizer_manager for more detail.
convert_to_variable: If `True` convert to variable in the
`GroupLassoBaseOpHandler`. If your graph creates variables outside of
`tf.get_variable`, set to `False`.
"""
conv2d_handler = conv2d_source_op_handler.Conv2DSourceOpHandler(
threshold, l1_fraction, convert_to_variable)
conv2d_transpose_handler = (
conv2d_transpose_source_op_handler.Conv2DTransposeSourceOpHandler(
threshold, l1_fraction, convert_to_variable))
matmul_handler = matmul_source_op_handler.MatMulSourceOpHandler(
threshold, l1_fraction, convert_to_variable)
if regularizer_decorator:
conv2d_handler = op_handler_decorator.OpHandlerDecorator(
conv2d_handler, regularizer_decorator, decorator_parameters)
conv2d_transpose_handler = op_handler_decorator.OpHandlerDecorator(
conv2d_transpose_handler, regularizer_decorator,
decorator_parameters)
matmul_handler = op_handler_decorator.OpHandlerDecorator(
matmul_handler, regularizer_decorator, decorator_parameters)
op_handler_dict = op_handlers.get_group_lasso_op_handler_dict()
op_handler_dict.update({
'Conv2D': conv2d_handler,
'Conv2DBackpropInput': conv2d_transpose_handler,
'MatMul': matmul_handler,
})
self._manager = orm.OpRegularizerManager(
output_boundary,
op_handler_dict,
input_boundary=input_boundary,
force_group=force_group,
regularizer_blacklist=regularizer_blacklist)
self._calculator = cost_calculator.CostCalculator(
self._manager,
resource_function.latency_function_factory(hardware, batch_size))
self._hardware = hardware
def __init__(self,
ops,
threshold,
l1_fraction=0,
regularizer_decorator: Type[
generic_regularizers.OpRegularizer] = None,
decorator_parameters=None,
force_group=None,
regularizer_blacklist=None,
convert_to_variable=True):
"""Creates a GroupLassoFlopsRegularizer object.
Args:
ops: A list of tf.Operation. An OpRegularizer will be created for all the
ops in `ops`, and recursively for all ops they depend on via data
dependency. Typically `ops` would contain a single tf.Operation, which
is the output of the network.
threshold: A float scalar, will be used as a 'threshold' for all
regularizer instances created by this class.
l1_fraction: Relative weight of L1 in L1 + L2 regularization.
regularizer_decorator: A class of OpRegularizer decorator to use.
decorator_parameters: A dictionary of parameters to pass to the decorator
factory. To be used only with decorators that requires parameters,
otherwise use None.
force_group: List of regex for ops that should be force-grouped. Each
regex corresponds to a separate group. Use '|' operator to specify
multiple patterns in a single regex. See op_regularizer_manager for more
detail.
regularizer_blacklist: List of regex for ops that should not be
regularized. See op_regularizer_manager for more detail.
convert_to_variable: If `True` convert to variable in the
`GroupLassoBaseOpHandler`. If you're graph creates variables outside of
`tf.get_variable`, set to `False`.
"""
conv2d_handler = conv2d_source_op_handler.Conv2DSourceOpHandler(
threshold, l1_fraction, convert_to_variable)
conv2d_transpose_handler = (
conv2d_transpose_source_op_handler.Conv2DTransposeSourceOpHandler(
threshold, l1_fraction, convert_to_variable))
matmul_handler = matmul_source_op_handler.MatMulSourceOpHandler(
threshold, l1_fraction, convert_to_variable)
if regularizer_decorator:
conv2d_handler = op_handler_decorator.OpHandlerDecorator(
conv2d_handler, regularizer_decorator, decorator_parameters)
conv2d_transpose_handler = op_handler_decorator.OpHandlerDecorator(
conv2d_transpose_handler, regularizer_decorator,
decorator_parameters)
matmul_handler = op_handler_decorator.OpHandlerDecorator(
matmul_handler, regularizer_decorator, decorator_parameters)
op_handler_dict = collections.defaultdict(
grouping_op_handler.GroupingOpHandler)
op_handler_dict.update({
'Conv2D':
conv2d_handler,
'Conv2DBackpropInput':
conv2d_transpose_handler,
'ConcatV2':
concat_op_handler.ConcatOpHandler(),
'DepthToSpace':
output_non_passthrough_op_handler.OutputNonPassthroughOpHandler(),
'DepthwiseConv2dNative':
depthwise_convolution_op_handler.DepthwiseConvolutionOpHandler(),
'MatMul':
matmul_handler,
'RandomUniform':
leaf_op_handler.LeafOpHandler(),
'Reshape':
leaf_op_handler.LeafOpHandler(),
'Shape':
leaf_op_handler.LeafOpHandler(),
'TensorArrayGatherV3':
leaf_op_handler.LeafOpHandler(),
'Transpose':
output_non_passthrough_op_handler.OutputNonPassthroughOpHandler(),
'StridedSlice':
leaf_op_handler.LeafOpHandler(),
})
self._manager = orm.OpRegularizerManager(
ops,
op_handler_dict,
force_group=force_group,
regularizer_blacklist=regularizer_blacklist)
self._calculator = cost_calculator.CostCalculator(
self._manager, resource_function.flop_function)
