def __init__(self, output_boundary: List[tf.Operation], threshold, l1_fraction=0.0, regularizer_decorator: Optional[Type[ generic_regularizers.OpRegularizer]] = None, decorator_parameters=None, input_boundary: Optional[List[tf.Operation]] = None, force_group: Optional[List[Text]] = None, regularizer_blacklist: Optional[List[Text]] = None): """Creates a GroupLassoModelSizeRegularizer object. Args: output_boundary: An OpRegularizer will be created for all these operations, and recursively for all ops they depend on via data dependency that does not involve ops from input_boundary. threshold: A float scalar, will be used as a 'threshold' for all regularizer instances created by this class. l1_fraction: A float scalar. The relative weight of L1 in L1 + L2 regularization. regularizer_decorator: A class of OpRegularizer decorator to use. decorator_parameters: A dictionary of parameters to pass to the decorator factory. To be used only with decorators that requires parameters, otherwise use None. input_boundary: A list of ops that represent the input boundary of the subgraph being regularized (input boundary is not regularized). force_group: List of regex for ops that should be force-grouped. Each regex corresponds to a separate group. Use '|' operator to specify multiple patterns in a single regex. See op_regularizer_manager for more detail. regularizer_blacklist: List of regex for ops that should not be regularized. See op_regularizer_manager for more detail. """ custom_handlers = { 'Conv2D': conv_handler.ConvSourceOpHandler(threshold, l1_fraction), 'Conv3D': conv_handler.ConvSourceOpHandler(threshold, l1_fraction), 'Conv2DBackpropInput': conv2d_transpose_handler.Conv2DTransposeSourceOpHandler( threshold, l1_fraction), 'MatMul': matmul_handler.MatMulSourceOpHandler(threshold, l1_fraction) } if regularizer_decorator: for key in custom_handlers: custom_handlers[key] = op_handler_decorator.OpHandlerDecorator( custom_handlers[key], regularizer_decorator, decorator_parameters) op_handler_dict = op_handlers.get_group_lasso_op_handler_dict() op_handler_dict.update(custom_handlers) self._manager = orm.OpRegularizerManager( output_boundary, op_handler_dict, input_boundary=input_boundary, force_group=force_group, regularizer_blacklist=regularizer_blacklist) self._calculator = cost_calculator.CostCalculator( self._manager, resource_function.model_size_function)
def __init__( self, ops, threshold, l1_fraction=0, regularizer_decorator: Type[generic_regularizers.OpRegularizer] = None, decorator_parameters=None, force_group=None, regularizer_blacklist=None): """Creates a GroupLassoActivationRegularizer object. Args: ops: A list of tf.Operation. An OpRegularizer will be created for all the ops in `ops`, and recursively for all ops they depend on via data dependency. Typically `ops` would contain a single tf.Operation, which is the output of the network. threshold: A float scalar, will be used as a 'threshold' for all regularizer instances created by this class. l1_fraction: Relative weight of L1 in L1 + L2 regularization. regularizer_decorator: A class of OpRegularizer decorator to use. decorator_parameters: A dictionary of parameters to pass to the decorator factory. To be used only with decorators that requires parameters, otherwise use None. force_group: List of regex for ops that should be force-grouped. Each regex corresponds to a separate group. Use '|' operator to specify multiple patterns in a single regex. See op_regularizer_manager for more detail. regularizer_blacklist: List of regex for ops that should not be regularized. See op_regularizer_manager for more detail. """ conv2d_handler = conv2d_source_op_handler.Conv2DSourceOpHandler( threshold, l1_fraction) conv2d_transpose_handler = ( conv2d_transpose_source_op_handler.Conv2DTransposeSourceOpHandler( threshold, l1_fraction)) matmul_handler = matmul_source_op_handler.MatMulSourceOpHandler( threshold, l1_fraction) if regularizer_decorator: conv2d_handler = op_handler_decorator.OpHandlerDecorator( conv2d_handler, regularizer_decorator, decorator_parameters) conv2d_transpose_handler = op_handler_decorator.OpHandlerDecorator( conv2d_transpose_handler, regularizer_decorator, decorator_parameters) matmul_handler = op_handler_decorator.OpHandlerDecorator( matmul_handler, regularizer_decorator, decorator_parameters) op_handler_dict = op_handlers.get_group_lasso_op_handler_dict() op_handler_dict.update({ 'Conv2D': conv2d_handler, 'Conv2DBackpropInput': conv2d_transpose_handler, 'MatMul': matmul_handler, }) self._manager = orm.OpRegularizerManager( ops, op_handler_dict, force_group=force_group, regularizer_blacklist=regularizer_blacklist) self._calculator = cost_calculator.CostCalculator( self._manager, resource_function.activation_count_function)
def testMatMul2D(self, size): inputs = tf.zeros((13, 2)) handler = matmul_source_op_handler.MatMulSourceOpHandler(0.1) kernel = tf.constant([[1, 2, 3], [4, 5, 6]], dtype=tf.float32) x = tf.matmul(inputs, kernel, transpose_b=False, name='MatMul') op_slice = orm.OpSlice(x.op, orm.Slice(0, size)) transpose_kernel = tf.constant([[1, 4], [2, 5], [3, 6]], dtype=tf.float32) x_other = tf.matmul( inputs, transpose_kernel, transpose_b=True, name='MatMulTransposedKernel') op_slice_other = orm.OpSlice(x_other.op, orm.Slice(0, size)) self.assertAllClose( handler.create_regularizer(op_slice).regularization_vector, handler.create_regularizer(op_slice_other).regularization_vector)
def __init__(self, output_boundary, threshold, hardware, batch_size=1, l1_fraction=0, regularizer_decorator=None, decorator_parameters=None, input_boundary=None, force_group=None, regularizer_blacklist=None, convert_to_variable=True): """Creates a GroupLassoFlopsRegularizer object. Args: output_boundary: An OpRegularizer will be created for all these operations, and recursively for all ops they depend on via data dependency that does not involve ops from input_boundary. threshold: A float scalar, will be used as a 'threshold' for all regularizer instances created by this class. hardware: String name of hardware platform to target. Must be a key from resource_function.PEAK_COMPUTE. batch_size: Integer batch size to calculate cost/loss for. l1_fraction: Relative weight of L1 in L1 + L2 regularization. regularizer_decorator: A class of OpRegularizer decorator to use. decorator_parameters: A dictionary of parameters to pass to the decorator factory. To be used only with decorators that requires parameters, otherwise use None. input_boundary: A list of ops that represent the input boundary of the subgraph being regularized (input boundary is not regularized). force_group: List of regex for ops that should be force-grouped. Each regex corresponds to a separate group. Use '|' operator to specify multiple patterns in a single regex. See op_regularizer_manager for more detail. regularizer_blacklist: List of regex for ops that should not be regularized. See op_regularizer_manager for more detail. convert_to_variable: If `True` convert to variable in the `GroupLassoBaseOpHandler`. If your graph creates variables outside of `tf.get_variable`, set to `False`. """ conv2d_handler = conv2d_source_op_handler.Conv2DSourceOpHandler( threshold, l1_fraction, convert_to_variable) conv2d_transpose_handler = ( conv2d_transpose_source_op_handler.Conv2DTransposeSourceOpHandler( threshold, l1_fraction, convert_to_variable)) matmul_handler = matmul_source_op_handler.MatMulSourceOpHandler( threshold, l1_fraction, convert_to_variable) if regularizer_decorator: conv2d_handler = op_handler_decorator.OpHandlerDecorator( conv2d_handler, regularizer_decorator, decorator_parameters) conv2d_transpose_handler = op_handler_decorator.OpHandlerDecorator( conv2d_transpose_handler, regularizer_decorator, decorator_parameters) matmul_handler = op_handler_decorator.OpHandlerDecorator( matmul_handler, regularizer_decorator, decorator_parameters) op_handler_dict = op_handlers.get_group_lasso_op_handler_dict() op_handler_dict.update({ 'Conv2D': conv2d_handler, 'Conv2DBackpropInput': conv2d_transpose_handler, 'MatMul': matmul_handler, }) self._manager = orm.OpRegularizerManager( output_boundary, op_handler_dict, input_boundary=input_boundary, force_group=force_group, regularizer_blacklist=regularizer_blacklist) self._calculator = cost_calculator.CostCalculator( self._manager, resource_function.latency_function_factory(hardware, batch_size)) self._hardware = hardware
def __init__(self, ops, threshold, l1_fraction=0, regularizer_decorator: Type[ generic_regularizers.OpRegularizer] = None, decorator_parameters=None, force_group=None, regularizer_blacklist=None, convert_to_variable=True): """Creates a GroupLassoFlopsRegularizer object. Args: ops: A list of tf.Operation. An OpRegularizer will be created for all the ops in `ops`, and recursively for all ops they depend on via data dependency. Typically `ops` would contain a single tf.Operation, which is the output of the network. threshold: A float scalar, will be used as a 'threshold' for all regularizer instances created by this class. l1_fraction: Relative weight of L1 in L1 + L2 regularization. regularizer_decorator: A class of OpRegularizer decorator to use. decorator_parameters: A dictionary of parameters to pass to the decorator factory. To be used only with decorators that requires parameters, otherwise use None. force_group: List of regex for ops that should be force-grouped. Each regex corresponds to a separate group. Use '|' operator to specify multiple patterns in a single regex. See op_regularizer_manager for more detail. regularizer_blacklist: List of regex for ops that should not be regularized. See op_regularizer_manager for more detail. convert_to_variable: If `True` convert to variable in the `GroupLassoBaseOpHandler`. If you're graph creates variables outside of `tf.get_variable`, set to `False`. """ conv2d_handler = conv2d_source_op_handler.Conv2DSourceOpHandler( threshold, l1_fraction, convert_to_variable) conv2d_transpose_handler = ( conv2d_transpose_source_op_handler.Conv2DTransposeSourceOpHandler( threshold, l1_fraction, convert_to_variable)) matmul_handler = matmul_source_op_handler.MatMulSourceOpHandler( threshold, l1_fraction, convert_to_variable) if regularizer_decorator: conv2d_handler = op_handler_decorator.OpHandlerDecorator( conv2d_handler, regularizer_decorator, decorator_parameters) conv2d_transpose_handler = op_handler_decorator.OpHandlerDecorator( conv2d_transpose_handler, regularizer_decorator, decorator_parameters) matmul_handler = op_handler_decorator.OpHandlerDecorator( matmul_handler, regularizer_decorator, decorator_parameters) op_handler_dict = collections.defaultdict( grouping_op_handler.GroupingOpHandler) op_handler_dict.update({ 'Conv2D': conv2d_handler, 'Conv2DBackpropInput': conv2d_transpose_handler, 'ConcatV2': concat_op_handler.ConcatOpHandler(), 'DepthToSpace': output_non_passthrough_op_handler.OutputNonPassthroughOpHandler(), 'DepthwiseConv2dNative': depthwise_convolution_op_handler.DepthwiseConvolutionOpHandler(), 'MatMul': matmul_handler, 'RandomUniform': leaf_op_handler.LeafOpHandler(), 'Reshape': leaf_op_handler.LeafOpHandler(), 'Shape': leaf_op_handler.LeafOpHandler(), 'TensorArrayGatherV3': leaf_op_handler.LeafOpHandler(), 'Transpose': output_non_passthrough_op_handler.OutputNonPassthroughOpHandler(), 'StridedSlice': leaf_op_handler.LeafOpHandler(), }) self._manager = orm.OpRegularizerManager( ops, op_handler_dict, force_group=force_group, regularizer_blacklist=regularizer_blacklist) self._calculator = cost_calculator.CostCalculator( self._manager, resource_function.flop_function)