def test_super_simple_norm(self): out_features = spaces.Categorical(12, 24, 36) bias = spaces.Categorical(True, False) model = super_core.SuperSequential( super_core.SuperSimpleNorm(5, 0.5), super_core.SuperLinear(10, out_features, bias=bias), ) print("The simple super module is:\n{:}".format(model)) model.apply_verbose(True) print(model.super_run_type) self.assertTrue(model[1].bias) inputs = torch.rand(20, 10) print("Input shape: {:}".format(inputs.shape)) outputs = model(inputs) self.assertEqual(tuple(outputs.shape), (20, 36)) abstract_space = model.abstract_search_space abstract_space.clean_last() abstract_child = abstract_space.random() print("The abstract searc space:\n{:}".format(abstract_space)) print("The abstract child program:\n{:}".format(abstract_child)) model.set_super_run_type(super_core.SuperRunMode.Candidate) model.apply_candidate(abstract_child) output_shape = (20, abstract_child["1"]["_out_features"].value) outputs = model(inputs) self.assertEqual(tuple(outputs.shape), output_shape)
def _create_stel(input_dim, output_dim): return super_core.SuperTransformerEncoderLayer( input_dim, output_dim, num_heads=spaces.Categorical(2, 4, 6), mlp_hidden_multiplier=spaces.Categorical(1, 2, 4), )
def test_super_mlp_v2(self): hidden_multiplier = spaces.Categorical(1.0, 2.0, 3.0) out_features = spaces.Categorical(24, 36, 48) mlp = super_core.SuperMLPv2(10, hidden_multiplier, out_features) print(mlp) mlp.apply_verbose(True) inputs = torch.rand(4, 10) outputs = mlp(inputs) self.assertEqual(tuple(outputs.shape), (4, 48)) abstract_space = mlp.abstract_search_space print( "The abstract search space for SuperMLPv2 is:\n{:}".format(abstract_space) ) abstract_space.clean_last() abstract_child = abstract_space.random(reuse_last=True) print("The abstract child program is:\n{:}".format(abstract_child)) mlp.set_super_run_type(super_core.SuperRunMode.Candidate) mlp.apply_candidate(abstract_child) outputs = mlp(inputs) output_shape = (4, abstract_child["_out_features"].value) self.assertEqual(tuple(outputs.shape), output_shape)
def test_super_mlp_v1(self): hidden_features = spaces.Categorical(12, 24, 36) out_features = spaces.Categorical(24, 36, 48) mlp = super_core.SuperMLPv1(10, hidden_features, out_features) print(mlp) mlp.apply_verbose(False) self.assertTrue(mlp.fc1._out_features, mlp.fc2._in_features) inputs = torch.rand(4, 10) outputs = mlp(inputs) self.assertEqual(tuple(outputs.shape), (4, 48)) abstract_space = mlp.abstract_search_space print("The abstract search space for SuperMLPv1 is:\n{:}".format( abstract_space)) self.assertEqual( abstract_space["fc1"]["_out_features"], abstract_space["fc2"]["_in_features"], ) self.assertTrue(abstract_space["fc1"]["_out_features"] is abstract_space["fc2"]["_in_features"]) abstract_space.clean_last() abstract_child = abstract_space.random(reuse_last=True) print("The abstract child program is:\n{:}".format(abstract_child)) self.assertEqual( abstract_child["fc1"]["_out_features"].value, abstract_child["fc2"]["_in_features"].value, ) mlp.set_super_run_type(super_core.SuperRunMode.Candidate) mlp.apply_candidate(abstract_child) outputs = mlp(inputs) output_shape = (4, abstract_child["fc2"]["_out_features"].value) self.assertEqual(tuple(outputs.shape), output_shape)
def test_super_attention(self): proj_dim = spaces.Categorical(12, 24, 36) num_heads = spaces.Categorical(2, 4, 6) model = super_core.SuperAttention(10, proj_dim, num_heads) print(model) model.apply_verbose(True) inputs = torch.rand(4, 20, 10) # batch size, sequence length, channel abstract_child, outputs = self._internal_func(inputs, model) output_shape = (4, 20, abstract_child["proj"]["_out_features"].value) self.assertEqual(tuple(outputs.shape), output_shape)
def test_super_sequential(batch, seq_dim, input_dim): out1_dim = spaces.Categorical(12, 24, 36) out2_dim = spaces.Categorical(24, 36, 48) out3_dim = spaces.Categorical(36, 72, 100) layer1 = _create_stel(input_dim, out1_dim) layer2 = _create_stel(out1_dim, out2_dim) layer3 = _create_stel(out2_dim, out3_dim) model = super_core.SuperSequential(layer1, layer2, layer3) print(model) model.apply_verbose(True) inputs = torch.rand(batch, seq_dim, input_dim) abstract_child, outputs = _internal_func(inputs, model) output_shape = ( batch, seq_dim, out3_dim.abstract(reuse_last=True).random(reuse_last=True).value, ) assert tuple(outputs.shape) == output_shape
def test_transformer_encoder(self, input_dim): output_dim = spaces.Categorical(12, 24, 36) model = super_core.SuperTransformerEncoderLayer( input_dim, output_dim=output_dim, num_heads=spaces.Categorical(2, 4, 6), mlp_hidden_multiplier=spaces.Categorical(1, 2, 4), ) print(model) model.apply_verbose(True) inputs = torch.rand(4, 20, input_dim) abstract_child, outputs = self._internal_func(inputs, model) output_shape = ( 4, 20, output_dim.abstract(reuse_last=True).random(reuse_last=True).value, ) self.assertEqual(tuple(outputs.shape), output_shape)
def test_super_stem(self): out_features = spaces.Categorical(24, 36, 48) model = super_core.SuperAlphaEBDv1(6, out_features) inputs = torch.rand(4, 360) abstract_space = model.abstract_search_space abstract_space.clean_last() abstract_child = abstract_space.random(reuse_last=True) print("The abstract searc space:\n{:}".format(abstract_space)) print("The abstract child program:\n{:}".format(abstract_child)) model.set_super_run_type(super_core.SuperRunMode.Candidate) model.apply_candidate(abstract_child) outputs = model(inputs) output_shape = (4, 60, abstract_child["_embed_dim"].value) self.assertEqual(tuple(outputs.shape), output_shape)
def _get_mul_specs(candidates, num): results = [] for i in range(num): results.append(spaces.Categorical(*candidates)) return results
for i in range(1, num + 1): results.append(i * multipler) return results def _assert_types(x, expected_types): if not isinstance(x, expected_types): raise TypeError("The type [{:}] is expected to be {:}.".format( type(x), expected_types)) DEFAULT_NET_CONFIG = None _default_max_depth = 5 DefaultSearchSpace = dict( d_feat=6, embed_dim=spaces.Categorical(*_get_list_mul(8, 16)), num_heads=_get_mul_specs((1, 2, 4, 8), _default_max_depth), mlp_hidden_multipliers=_get_mul_specs((0.5, 1, 2, 4, 8), _default_max_depth), qkv_bias=True, pos_drop=0.0, other_drop=0.0, ) class SuperTransformer(super_core.SuperModule): """The super model for transformer.""" def __init__( self, d_feat: int = 6, embed_dim: List[