Пример #1
0
    def test_super_simple_norm(self):
        out_features = spaces.Categorical(12, 24, 36)
        bias = spaces.Categorical(True, False)
        model = super_core.SuperSequential(
            super_core.SuperSimpleNorm(5, 0.5),
            super_core.SuperLinear(10, out_features, bias=bias),
        )
        print("The simple super module is:\n{:}".format(model))
        model.apply_verbose(True)

        print(model.super_run_type)
        self.assertTrue(model[1].bias)

        inputs = torch.rand(20, 10)
        print("Input shape: {:}".format(inputs.shape))
        outputs = model(inputs)
        self.assertEqual(tuple(outputs.shape), (20, 36))

        abstract_space = model.abstract_search_space
        abstract_space.clean_last()
        abstract_child = abstract_space.random()
        print("The abstract searc space:\n{:}".format(abstract_space))
        print("The abstract child program:\n{:}".format(abstract_child))

        model.set_super_run_type(super_core.SuperRunMode.Candidate)
        model.apply_candidate(abstract_child)

        output_shape = (20, abstract_child["1"]["_out_features"].value)
        outputs = model(inputs)
        self.assertEqual(tuple(outputs.shape), output_shape)
def _create_stel(input_dim, output_dim):
    return super_core.SuperTransformerEncoderLayer(
        input_dim,
        output_dim,
        num_heads=spaces.Categorical(2, 4, 6),
        mlp_hidden_multiplier=spaces.Categorical(1, 2, 4),
    )
Пример #3
0
    def test_super_mlp_v2(self):
        hidden_multiplier = spaces.Categorical(1.0, 2.0, 3.0)
        out_features = spaces.Categorical(24, 36, 48)
        mlp = super_core.SuperMLPv2(10, hidden_multiplier, out_features)
        print(mlp)
        mlp.apply_verbose(True)

        inputs = torch.rand(4, 10)
        outputs = mlp(inputs)
        self.assertEqual(tuple(outputs.shape), (4, 48))

        abstract_space = mlp.abstract_search_space
        print(
            "The abstract search space for SuperMLPv2 is:\n{:}".format(abstract_space)
        )

        abstract_space.clean_last()
        abstract_child = abstract_space.random(reuse_last=True)
        print("The abstract child program is:\n{:}".format(abstract_child))

        mlp.set_super_run_type(super_core.SuperRunMode.Candidate)
        mlp.apply_candidate(abstract_child)
        outputs = mlp(inputs)
        output_shape = (4, abstract_child["_out_features"].value)
        self.assertEqual(tuple(outputs.shape), output_shape)
Пример #4
0
    def test_super_mlp_v1(self):
        hidden_features = spaces.Categorical(12, 24, 36)
        out_features = spaces.Categorical(24, 36, 48)
        mlp = super_core.SuperMLPv1(10, hidden_features, out_features)
        print(mlp)
        mlp.apply_verbose(False)
        self.assertTrue(mlp.fc1._out_features, mlp.fc2._in_features)

        inputs = torch.rand(4, 10)
        outputs = mlp(inputs)
        self.assertEqual(tuple(outputs.shape), (4, 48))

        abstract_space = mlp.abstract_search_space
        print("The abstract search space for SuperMLPv1 is:\n{:}".format(
            abstract_space))
        self.assertEqual(
            abstract_space["fc1"]["_out_features"],
            abstract_space["fc2"]["_in_features"],
        )
        self.assertTrue(abstract_space["fc1"]["_out_features"] is
                        abstract_space["fc2"]["_in_features"])

        abstract_space.clean_last()
        abstract_child = abstract_space.random(reuse_last=True)
        print("The abstract child program is:\n{:}".format(abstract_child))
        self.assertEqual(
            abstract_child["fc1"]["_out_features"].value,
            abstract_child["fc2"]["_in_features"].value,
        )

        mlp.set_super_run_type(super_core.SuperRunMode.Candidate)
        mlp.apply_candidate(abstract_child)
        outputs = mlp(inputs)
        output_shape = (4, abstract_child["fc2"]["_out_features"].value)
        self.assertEqual(tuple(outputs.shape), output_shape)
Пример #5
0
    def test_super_attention(self):
        proj_dim = spaces.Categorical(12, 24, 36)
        num_heads = spaces.Categorical(2, 4, 6)
        model = super_core.SuperAttention(10, proj_dim, num_heads)
        print(model)
        model.apply_verbose(True)

        inputs = torch.rand(4, 20, 10)  # batch size, sequence length, channel
        abstract_child, outputs = self._internal_func(inputs, model)
        output_shape = (4, 20, abstract_child["proj"]["_out_features"].value)
        self.assertEqual(tuple(outputs.shape), output_shape)
def test_super_sequential(batch, seq_dim, input_dim):
    out1_dim = spaces.Categorical(12, 24, 36)
    out2_dim = spaces.Categorical(24, 36, 48)
    out3_dim = spaces.Categorical(36, 72, 100)
    layer1 = _create_stel(input_dim, out1_dim)
    layer2 = _create_stel(out1_dim, out2_dim)
    layer3 = _create_stel(out2_dim, out3_dim)
    model = super_core.SuperSequential(layer1, layer2, layer3)
    print(model)
    model.apply_verbose(True)
    inputs = torch.rand(batch, seq_dim, input_dim)
    abstract_child, outputs = _internal_func(inputs, model)
    output_shape = (
        batch,
        seq_dim,
        out3_dim.abstract(reuse_last=True).random(reuse_last=True).value,
    )
    assert tuple(outputs.shape) == output_shape
Пример #7
0
 def test_transformer_encoder(self, input_dim):
     output_dim = spaces.Categorical(12, 24, 36)
     model = super_core.SuperTransformerEncoderLayer(
         input_dim,
         output_dim=output_dim,
         num_heads=spaces.Categorical(2, 4, 6),
         mlp_hidden_multiplier=spaces.Categorical(1, 2, 4),
     )
     print(model)
     model.apply_verbose(True)
     inputs = torch.rand(4, 20, input_dim)
     abstract_child, outputs = self._internal_func(inputs, model)
     output_shape = (
         4,
         20,
         output_dim.abstract(reuse_last=True).random(reuse_last=True).value,
     )
     self.assertEqual(tuple(outputs.shape), output_shape)
Пример #8
0
    def test_super_stem(self):
        out_features = spaces.Categorical(24, 36, 48)
        model = super_core.SuperAlphaEBDv1(6, out_features)
        inputs = torch.rand(4, 360)

        abstract_space = model.abstract_search_space
        abstract_space.clean_last()
        abstract_child = abstract_space.random(reuse_last=True)
        print("The abstract searc space:\n{:}".format(abstract_space))
        print("The abstract child program:\n{:}".format(abstract_child))

        model.set_super_run_type(super_core.SuperRunMode.Candidate)
        model.apply_candidate(abstract_child)
        outputs = model(inputs)
        output_shape = (4, 60, abstract_child["_embed_dim"].value)
        self.assertEqual(tuple(outputs.shape), output_shape)
Пример #9
0
def _get_mul_specs(candidates, num):
    results = []
    for i in range(num):
        results.append(spaces.Categorical(*candidates))
    return results
Пример #10
0
    for i in range(1, num + 1):
        results.append(i * multipler)
    return results


def _assert_types(x, expected_types):
    if not isinstance(x, expected_types):
        raise TypeError("The type [{:}] is expected to be {:}.".format(
            type(x), expected_types))


DEFAULT_NET_CONFIG = None
_default_max_depth = 5
DefaultSearchSpace = dict(
    d_feat=6,
    embed_dim=spaces.Categorical(*_get_list_mul(8, 16)),
    num_heads=_get_mul_specs((1, 2, 4, 8), _default_max_depth),
    mlp_hidden_multipliers=_get_mul_specs((0.5, 1, 2, 4, 8),
                                          _default_max_depth),
    qkv_bias=True,
    pos_drop=0.0,
    other_drop=0.0,
)


class SuperTransformer(super_core.SuperModule):
    """The super model for transformer."""
    def __init__(
        self,
        d_feat: int = 6,
        embed_dim: List[