def test_block_conversions(self):
from ipu_sparse_ops import sparse
a = np.kron([[1, 0], [1, 0]], [[1, 2], [3, 4]])
b = np.kron([[0, 0], [1, 0]], [[4, 4], [4, 4]])
dense = a + b
bs = 2
spec = sparse.matmul_spec_from_max(2 * bs, [1, 2 * bs],
2,
block_size=bs,
dtype=tf.float32)
blocks = np.reshape([1, 2, 3, 4, 5, 6, 7, 8], [2, bs, bs])
t = ([0, 1], [0, 0], blocks)
n = sparse.dense_from_triplets(spec, *t)
assert_equal(dense, n)
# Check that mask from dense and mask from triplets
# return the same result:
mask_dense = np.zeros_like(dense)
mask_dense[np.nonzero(dense)] = 1
mask_trips = sparse.mask_from_triplets(spec, *t)
assert_equal(mask_dense, mask_trips)
# Check triplets from dense returns same triplets:
td = sparse.triplets_from_dense(dense, bs)
assert_equal(t[0], td.row_indices)
assert_equal(t[1], td.col_indices)
assert_equal(t[2], td.values)
def make_triplets_test_inputs(args):
input_size = args.input_size
output_size = args.output_size
batch_size = args.batch_size
weights_type = tf.float16 if args.data_type == 'fp16' else tf.float32
if args.pattern == 'fixed':
rhs_values = np.random.rand(input_size, output_size)
sparse_mask = np.identity(input_size)
sparse_mask[1, 3] = 1
sparse_mask[0, 7] = 1
masked_rhs = np.multiply(sparse_mask[:, 0:output_size], rhs_values)
triplets = sparse.triplets_from_dense(masked_rhs)
fc = layers.SparseFcLayer.from_triplets(
args.output_size, [args.batch_size, args.input_size], *triplets,
matmul_options={"metaInfoBucketOversizeProportion": 0.1},
name='sparse_fc_from_triplets',
dtype=weights_type,
bias=False, relu=False)
elif args.pattern == 'random_sign_ones':
indices_random_gen = np.random.default_rng(seed=random_seed)
fc = layers.SparseFcLayer.from_random_generator(
args.output_size, [args.batch_size, args.input_size], args.density,
random_sign_ones_generator, indices_random_gen,
matmul_options={"metaInfoBucketOversizeProportion": 0.1},
name='sparse_fc_from_random_sign_ones', bias=False, relu=False)
masked_rhs = sparse.dense_from_triplets(fc.weights.spec, *fc.weights.triplets)
elif args.pattern == "random_orthogonal":
fc = layers.SparseFcLayer.from_random_orthonormal_generator(
args.output_size, [args.batch_size, args.input_size], args.density,
matmul_options={"metaInfoBucketOversizeProportion": 0.1},
name='sparse_fc_from_random_orthogonal', dtype=weights_type,
bias=False, relu=False)
masked_rhs = sparse.dense_from_triplets(fc.weights.spec, *fc.weights.triplets)
else:
random_gen = np.random.default_rng(seed=random_seed)
indices_random_gen = np.random.default_rng(seed=random_seed)
fc = layers.SparseFcLayer.from_random_generator(
args.output_size, [args.batch_size, args.input_size], args.density,
random_gen.standard_normal, indices_random_gen,
matmul_options={"metaInfoBucketOversizeProportion": 0.1},
name='sparse_fc_from_random',
dtype=weights_type,
bias=False, relu=False)
masked_rhs = sparse.dense_from_triplets(fc.weights.spec, *fc.weights.triplets)
return fc, masked_rhs.astype(weights_type.as_numpy_dtype())
def test_conversions(self):
from ipu_sparse_ops import sparse
m = np.array([[10, 0], [0, 20]])
t = sparse.triplets_from_dense(m)
assert_equal(t[0], [0, 1])
assert_equal(t[1], [0, 1])
assert_equal(t[2], [10, 20])
spec = sparse.matmul_spec_from_max(2, [1, 2], 2, tf.float32)
n = sparse.dense_from_triplets(spec, *t)
assert_equal(n, m)
o = sparse.mask_from_triplets(spec, *t)
assert_equal(o, np.array([[1, 0], [0, 1]]))
def test_representation_round_trip_elements(self):
from ipu_sparse_ops import sparse
bs = 16
block_mask = np.array([[1, 0, 0], [0, 1, 0], [1, 1, 0], [0, 0, 1]])
mask = np.kron(block_mask, np.ones(shape=[bs, bs])).astype(int)
n_els = np.count_nonzero(mask)
dense = np.zeros_like(mask)
dense[np.nonzero(mask)] = np.arange(n_els)
opts = {"metaInfoBucketOversizeProportion": 1}
t = sparse.triplets_from_dense(dense)
spec = sparse.matmul_spec_from_max(dense.shape[1], [2, dense.shape[0]],
max_non_zeros=n_els,
block_size=1,
dtype=tf.float32)
r = sparse.representation_from_triplets(spec, *t, opts)
t_rt = sparse.triplets_from_representation(spec, r, opts)
dense_rt = sparse.dense_from_triplets(spec, *t_rt)
assert_equal(dense, dense_rt)
def test_representation_round_trip_blocks(self):
from ipu_sparse_ops import sparse
for bs in [4, 8, 16]:
# Create a mask that describes the non-zero block structure:
block_mask = np.array([[1, 1, 0], [0, 1, 0], [1, 0, 0], [0, 0, 1]])
n_blocks = np.count_nonzero(block_mask)
# From that produce an element-wise mask using a Kronecker product:
mask = np.kron(block_mask, np.ones(shape=[bs, bs])).astype(int)
n_els = np.count_nonzero(mask)
# Make a dense matrix from the element-wise mask and fill with random values:
dense = np.zeros_like(mask, dtype=np.float32)
values = np.random.rand(n_els)
dense[np.nonzero(mask)] = values
# Make the spec for the sparse matmul:
opts = {"metaInfoBucketOversizeProportion": 1}
spec = sparse.matmul_spec_from_max(dense.shape[1],
[2, dense.shape[0]],
max_non_zeros=n_blocks,
block_size=bs,
dtype=tf.float32)
# Make triplets indices from the block mask:
t = sparse.triplets_from_dense(block_mask)
# Then fill in triplet's values by extracting the blocks
# from the dense matrix (this can't be done by reshaping):
t_block = sparse.Triplets(
t.row_indices, t.col_indices,
sparse.blocks_at_indices(t.row_indices, t.col_indices, bs,
dense))
# Convert to on device representation and back and check the
# result is the dense matrix we sytarted with:
r = sparse.representation_from_triplets(spec, *t_block, opts)
t_rt = sparse.triplets_from_representation(spec, r, opts)
dense_rt = sparse.dense_from_triplets(spec, *t_rt)
assert_equal(dense, dense_rt)
# Check triplets from dense returns original triplets:
td = sparse.triplets_from_dense(dense_rt, bs)
assert_equal(t_block.row_indices, td.row_indices)
assert_equal(t_block.col_indices, td.col_indices)
assert_equal(t_block.values, td.values)
def make_triplets_test_inputs(args, data_type):
input_size = args.input_size
output_size = args.output_size
batch_size = args.batch_size
num_groups = 1
topk_ratio = 0.5
if args.pattern == 'fixed':
rhs_values = np.random.rand(input_size, output_size)
sparse_mask = np.identity(input_size)
sparse_mask[1, 3] = 1
sparse_mask[0, 7] = 1
masked_rhs = np.multiply(sparse_mask[:, 0:output_size], rhs_values)
triplets = sparse.triplets_from_dense(masked_rhs)
fc = layers.SparseFcLayer.from_triplets(
args.output_size, [args.batch_size, args.input_size], topk_ratio,
*triplets)
else:
random_gen = np.random.default_rng(seed=random_seed)
fc = layers.SparseFcLayer.from_random_generator(
args.output_size, [args.batch_size, args.input_size], args.density,
topk_ratio, random_gen.standard_normal, random_seed)
masked_rhs = sparse.dense_from_triplets(fc.spec, *fc.triplets)
return fc, masked_rhs
def prune_and_grow(name,
triplets,
shape,
spec,
max_non_zeros,
slot_triplets,
prune_schedule,
prune_ratio: float,
grad_w: np.array,
grow_method='rigl',
random_gen=None,
ipu_pooling_type='NONE'):
"""
Performs the pruning and growing of the weights to update the sparsity pattern, for the current fc layer
:param name: A debug name
:param triplets: Current triplets to prune and grow
:param shape: Shape of the dense matrix
:param spec: Specs of the sparse matmul
:param max_non_zeros: Maximum number of non-zeros values
:slot_triplets: Triplets for the current slots
:param prune_schedule: a function which given max prune count returns the number of weights to update
:param prune_ratio: the maximum percentage of this layer's weights to update
:param grad_w: A numpy array containing the dense gradients for each sub-layer
:param grow_method: Method used to regrow the weights, either rigl or random selection
:param random_gen: Random number generator to be used if the grow_method is 'random'
"""
if isinstance(grad_w, list):
grad_w = grad_w[0]
if isinstance(grad_w, dict):
grad_w = [grad for grad in grad_w.values()][0]
# Compute the prune count using the provides schedule, the max number of zeros in the
# layer and the pruning ratio
prune_count = prune_schedule(
max_pruned=int(np.ceil(prune_ratio * max_non_zeros)))
if prune_count == 0:
logger.info("Nothing to prune according to prune schedule.")
return None
logger.info(
f"Triplet stats before prune and grow for {name}: {sparse.triplet_stats(*triplets)}"
)
if logger.level <= logging.DEBUG:
abs_nz_values = np.abs(triplets[2])
if len(abs_nz_values.shape) > 1:
abs_nz_values = abs_nz_values.sum(-1).sum(-1)
block_input_size = spec.input_size // spec.block_size
block_output_size = spec.output_size // spec.block_size
block_spec = sparse.MatmulSpec(
block_size=1,
input_size=block_input_size,
output_size=block_output_size,
num_groups=spec.num_groups,
batch_size=spec.batch_size,
data_type=spec.data_type,
max_non_zero_blocks=spec.max_non_zero_blocks,
pooling_type=spec.pooling_type)
dense_abs_weights = sparse.dense_from_triplets(
block_spec, triplets[0], triplets[1], abs_nz_values)
plot_and_log_matrix(name + "/abs_block_weights", dense_abs_weights)
# Prune bottom k weights
logging.debug(
f"Pruning and grow also applies to these slot vars: {slot_triplets.keys()}"
)
slot_values = {
name: triplet.values
for name, triplet in slot_triplets.items()
}
remaining_triplets, remaining_slot_values = prune_bottom_k_weights(
*triplets, slot_values, prune_count, name)
# regrow weights
logger.debug(
f"Regrowing non-zeros for layer {name} using '{grow_method}' method.")
if grow_method == 'rigl':
weights_shape = np.array(triplets[2]).shape
block_size = 1 if len(weights_shape) == 1 else weights_shape[-1]
# Grow back new indices using Rig-L: (https://arxiv.org/abs/1911.11134)
if (shape != grad_w.shape
and (ipu_pooling_type == "NONE" or block_size == 1)):
raise RuntimeError(
f"Dense weight gradient has unexpected shape.Expected {shape}, got {grad_w.shape}"
)
new_triplets = regrow_rigl(triplets, grad_w, zero_values_generator,
prune_count, ipu_pooling_type == "NONE",
name)
if grow_method == 'random':
# Random replacement strategy: add back random indices
# Gen some replacement random indices excluding all the existing
# ones then we will swap for the pruned ones:
new_triplets = sparse.random_triplets(
spec,
indices_initialiser_gen=random_gen,
value_generator=zero_values_generator,
excluded_indices=(triplets[0], triplets[1]),
count=prune_count)
grown_triplets, grown_slots = join_triplets(remaining_triplets,
new_triplets,
remaining_slot_values,
prune_count)
if len(grown_triplets[0]) != max_non_zeros:
raise ValueError(
f"Grown row count {len(grown_triplets[0])} does not match expected count {max_non_zeros}"
)
if len(grown_triplets[1]) != max_non_zeros:
raise ValueError(
f"Grown col count {len(grown_triplets[1])} does not match expected count {max_non_zeros}"
)
if len(grown_triplets[2]) != max_non_zeros:
raise ValueError(
f"Grown col count {len(grown_triplets[2])} does not match expected count {max_non_zeros}"
)
for grown_slot in grown_slots.values():
if len(grown_slot) != max_non_zeros:
raise ValueError(
f"Grown col count {len(grown_slot)} does not match expected count {max_non_zeros}"
)
grown_triplets = sparse.Triplets(grown_triplets[0], grown_triplets[1],
grown_triplets[2])
grown_slots = {
name: sparse.Triplets(grown_triplets[0], grown_triplets[1], grown_slot)
for name, grown_slot in grown_slots.items()
}
logger.info(
f"Triplet stats after prune and grow for {name}: {sparse.triplet_stats(*grown_triplets)}"
)
return {
'gt': grown_triplets,
'nt': new_triplets,
'rt': remaining_triplets,
'gs': grown_slots,
'name': name
}
def make_fc_layer_and_test_inputs(args):
input_size = args.input_size
output_size = args.output_size
batch_size = args.batch_size
weights_type = tf.float16 if args.data_type == 'fp16' else tf.float32
matmul_opts = {"metaInfoBucketOversizeProportion": args.meta_info_oversize}
if args.pattern == 'fixed':
in_blocks = input_size // args.block_size
out_blocks = output_size // args.block_size
identity_size = max(in_blocks, out_blocks)
block_mask = np.identity(identity_size)[0:in_blocks, 0:out_blocks]
block_mask[1, 3] = 1
block_mask[0, 7] = 1
n_blocks = np.count_nonzero(block_mask)
el_mask = sparse.block_mask_to_element(block_mask, args.block_size)
n_els = np.count_nonzero(el_mask)
masked_rhs = np.zeros_like(el_mask, dtype=np.float32)
values = np.random.rand(n_els)
masked_rhs[np.nonzero(el_mask)] = values
if args.block_size == 1:
triplets = sparse.triplets_from_dense(masked_rhs)
else:
triplets = sparse.triplets_from_dense(block_mask)
triplets = sparse.Triplets(
triplets.row_indices, triplets.col_indices,
sparse.blocks_at_indices(triplets.row_indices,
triplets.col_indices, args.block_size,
masked_rhs))
fc = layers.SparseFcLayer.from_triplets(
args.output_size, [args.batch_size, args.input_size],
*triplets,
matmul_options=matmul_opts,
name='sparse_fc_from_triplets',
dtype=weights_type,
use_bias=False,
relu=False,
pooling_type=args.pooling_type)
elif args.pattern == 'random_sign_ones':
indices_random_gen = np.random.default_rng(seed=random_seed)
fc = layers.SparseFcLayer.from_random_generator(
args.output_size, [args.batch_size, args.input_size],
args.density,
args.block_size,
random_sign_ones_generator,
indices_random_gen,
matmul_options=matmul_opts,
name='sparse_fc_from_random_sign_ones',
use_bias=False,
relu=False,
pooling_type=args.pooling_type)
masked_rhs = sparse.dense_from_triplets(fc.weights.spec,
*fc.weights.triplets)
elif args.pattern == "random_orthogonal":
if args.input_size != args.output_size:
raise ValueError(
"random_orthogonal pattern requires square matrix")
matrix, max_non_zeros = sparse.gen_sparse_rand_orthog_mat(
args.output_size, args.density, args.block_size)
triplets = sparse.triplets_from_dense(matrix, args.block_size)
fc = layers.SparseFcLayer.from_triplets(
args.output_size, [args.batch_size, args.input_size],
*triplets,
matmul_options=matmul_opts,
name='sparse_fc_random_orthogonal',
dtype=weights_type,
use_bias=False,
relu=False,
pooling_type=args.pooling_type)
masked_rhs = sparse.dense_from_triplets(fc.weights.spec,
*fc.weights.triplets)
else:
random_gen = np.random.default_rng(seed=random_seed)
indices_random_gen = np.random.default_rng(seed=random_seed)
fc = layers.SparseFcLayer.from_random_generator(
args.output_size, [args.batch_size, args.input_size],
args.density,
args.block_size,
random_gen.standard_normal,
indices_random_gen,
matmul_options=matmul_opts,
name='sparse_fc_from_random',
dtype=weights_type,
use_bias=False,
relu=False,
pooling_type=args.pooling_type)
masked_rhs = fc.weights.extract_dense()
return fc, masked_rhs.astype(weights_type.as_numpy_dtype())
def extract_dense(self):
return sparse.dense_from_triplets(self.spec, *self.triplets)
def extract_dense(self) -> np.ndarray:
return sparse.dense_from_triplets(self.spec, *self.triplets)