def test_super_simple_learn_norm(self):
out_features = spaces.Categorical(12, 24, 36)
bias = spaces.Categorical(True, False)
model = super_core.SuperSequential(
super_core.SuperSimpleLearnableNorm(),
super_core.SuperIdentity(),
super_core.SuperLinear(10, out_features, bias=bias),
)
print("The simple super module is:\n{:}".format(model))
model.apply_verbose(True)
print(model.super_run_type)
self.assertTrue(model[2].bias)
inputs = torch.rand(20, 10)
print("Input shape: {:}".format(inputs.shape))
outputs = model(inputs)
self.assertEqual(tuple(outputs.shape), (20, 36))
abstract_space = model.abstract_search_space
abstract_space.clean_last()
abstract_child = abstract_space.random()
print("The abstract searc space:\n{:}".format(abstract_space))
print("The abstract child program:\n{:}".format(abstract_child))
model.set_super_run_type(super_core.SuperRunMode.Candidate)
model.enable_candidate()
model.apply_candidate(abstract_child)
output_shape = (20, abstract_child["2"]["_out_features"].value)
outputs = model(inputs)
self.assertEqual(tuple(outputs.shape), output_shape)
def _create_stel(input_dim, output_dim, order):
return super_core.SuperSequential(
super_core.SuperLinear(input_dim, output_dim),
super_core.SuperTransformerEncoderLayer(
output_dim,
num_heads=spaces.Categorical(2, 4, 6),
mlp_hidden_multiplier=spaces.Categorical(1, 2, 4),
order=order,
),
)
def test_super_attention(self):
proj_dim = spaces.Categorical(12, 24, 36)
num_heads = spaces.Categorical(2, 4, 6)
model = super_core.SuperSelfAttention(10, proj_dim, num_heads)
print(model)
model.apply_verbose(True)
inputs = torch.rand(4, 20, 10) # batch size, sequence length, channel
abstract_child, outputs = self._internal_func(inputs, model)
output_shape = (4, 20, abstract_child["proj"]["_out_features"].value)
self.assertEqual(tuple(outputs.shape), output_shape)
def test_super_sequential(batch, seq_dim, input_dim, order):
out1_dim = spaces.Categorical(12, 24, 36)
out2_dim = spaces.Categorical(24, 36, 48)
out3_dim = spaces.Categorical(36, 72, 100)
layer1 = _create_stel(input_dim, out1_dim, order)
layer2 = _create_stel(out1_dim, out2_dim, order)
layer3 = _create_stel(out2_dim, out3_dim, order)
model = super_core.SuperSequential(layer1, layer2, layer3)
print(model)
model.apply_verbose(True)
inputs = torch.rand(batch, seq_dim, input_dim)
abstract_child, outputs = _internal_func(inputs, model)
output_shape = (
batch,
seq_dim,
out3_dim.abstract(reuse_last=True).random(reuse_last=True).value,
)
assert tuple(outputs.shape) == output_shape
def test_transformer_encoder(self, input_dim):
output_dim = spaces.Categorical(12, 24, 36)
model = super_core.SuperSequential(
super_core.SuperLinear(input_dim, output_dim),
super_core.SuperTransformerEncoderLayer(
output_dim,
num_heads=spaces.Categorical(2, 4, 6),
mlp_hidden_multiplier=spaces.Categorical(1, 2, 4),
),
)
print(model)
model.apply_verbose(True)
inputs = torch.rand(4, 20, input_dim)
abstract_child, outputs = self._internal_func(inputs, model)
output_shape = (
4,
20,
output_dim.abstract(reuse_last=True).random(reuse_last=True).value,
)
self.assertEqual(tuple(outputs.shape), output_shape)
def _get_mul_specs(candidates, num):
results = []
for i in range(num):
results.append(spaces.Categorical(*candidates))
return results