Beispiel #1
0
def transform_gpt2_to_texar_config(input_json_path):
    """
    Remap the config file
    """
    config_gpt = json.loads(open(input_json_path).read())
    configs = dict()
    configs["vocab_size"] = config_gpt["n_vocab"]
    configs["context_size"] = config_gpt["n_ctx"]
    configs["embedding_size"] = config_gpt["n_embd"]
    hidden_dim = config_gpt["n_embd"]
    configs["embed"] = {
        "dim": hidden_dim,
    }
    configs["position_size"] = config_gpt["n_ctx"]
    configs["pos_embed"] = {"dim": hidden_dim}
    configs["decoder"] = {
        "dim": hidden_dim,
        "num_blocks": config_gpt["n_layer"],
        "use_gpt_config": True,
        "embedding_dropout": 0,
        "residual_dropout": 0,
        "multihead_attention": {
            "use_bias": True,
            "num_units": hidden_dim,
            "num_heads": config_gpt["n_head"],
            "output_dim": hidden_dim,
        },
        "initializer": {
            "type": "variance_scaling_initializer",
            "kwargs": {
                "scale": 1.0,
                "mode": "FAN_AVG",
                "uniform": True,
            },
        },
        "poswise_feedforward": {
            "layers": [{
                "type": "Linear",
                "kwargs": {
                    "in_features": hidden_dim,
                    "out_features": hidden_dim * 4,
                    "bias": True,
                }
            }, {
                "type": "GPTGELU",
                "kwargs": {}
            }, {
                "type": "Linear",
                "kwargs": {
                    "in_features": hidden_dim * 4,
                    "out_features": hidden_dim,
                    "bias": True,
                }
            }],
            "name":
            "ffn",
        },
    }
    return HParams(configs, default_hparams=None)
Beispiel #2
0
def transform_gpt2_to_texar_config(input_json_path):
    """
    Remap the config file
    """
    config_gpt = json.loads(open(input_json_path).read())
    configs = dict()
    configs["vocab_size"] = config_gpt["n_vocab"]
    configs["context_size"] = config_gpt["n_ctx"]
    configs["embedding_size"] = config_gpt["n_embd"]
    hidden_dim = config_gpt["n_embd"]
    configs['embed'] = {
        'dim': hidden_dim,
    }
    configs['position_size'] = config_gpt['n_ctx'],
    configs['pos_embed'] = {
        'dim': hidden_dim
    }
    configs['decoder'] = {
        'dim': hidden_dim,
        'num_blocks': config_gpt['n_layer'],
        'multihead_attention': {
            'use_bias': True,
            'num_units': hidden_dim,
            'num_heads': config_gpt['n_head'],
            'output_dim': hidden_dim,
        },
        'initializer': {
            'type': 'variance_scaling_initializer',
            'kwargs': {
                'scale': 1.0,
                'mode': 'fan_avg',
                'distribution': 'uniform',
            },
        },
        'poswise_feedforward': {
            "layers": [
                {
                    "type": "Dense",
                    "kwargs": {
                        "name": "conv1",
                        "units": hidden_dim*4,
                        "activation": "gelu",
                        "use_bias": True,
                    }
                },
                {
                    "type": "Dense",
                    "kwargs": {
                        "name": "conv2",
                        "units": hidden_dim,
                        "use_bias": True,
                    }
                }
            ],
            "name": "ffn",
        },
    }
    return HParams(configs, default_hparams=None)
Beispiel #3
0
def transform_bert_to_texar_config(input_json):
    config_ckpt = json.loads(open(input_json).read())
    configs = {}
    configs['random_seed'] = 123
    configs['hidden_size'] = config_ckpt['hidden_size']
    hidden_dim = config_ckpt['hidden_size']
    configs['embed'] = {'name': 'word_embeddings', 'dim': hidden_dim}
    configs['vocab_size'] = config_ckpt['vocab_size']
    configs['segment_embed'] = {
        'name': 'token_type_embeddings',
        'dim': hidden_dim
    }
    configs['type_vocab_size'] = config_ckpt['type_vocab_size']

    configs['encoder'] = {
        'name': 'encoder',
        'position_embedder_type': 'variables',
        'position_size': config_ckpt['max_position_embeddings'],
        'position_embedder_hparams': {
            'dim': hidden_dim,
        },
        'embedding_dropout': config_ckpt['hidden_dropout_prob'],
        'num_blocks': config_ckpt['num_hidden_layers'],
        'multihead_attention': {
            'use_bias': True,
            'num_units': hidden_dim,
            'num_heads': config_ckpt['num_attention_heads'],
            'output_dim': hidden_dim,
            'dropout_rate': config_ckpt['attention_probs_dropout_prob'],
            'name': 'self'
        },
        'residual_dropout': config_ckpt['hidden_dropout_prob'],
        'dim': hidden_dim,
        'use_bert_config': True,
        'poswise_feedforward': {
            "layers": [
                {
                    'type': 'Dense',
                    'kwargs': {
                        'name': 'intermediate',
                        'units': config_ckpt['intermediate_size'],
                        'activation': config_ckpt['hidden_act'],
                        'use_bias': True,
                    }
                },
                {
                    'type': 'Dense',
                    'kwargs': {
                        'name': 'output',
                        'units': hidden_dim,
                        'activation': None,
                        'use_bias': True,
                    }
                },
            ],
        },
    }
    return HParams(configs, default_hparams=None)
Beispiel #4
0
def transform_gpt2_to_texar_config(input_json_path):
    """
    Remap the config file
    """
    config_gpt = json.loads(open(input_json_path).read())
    configs = dict()
    configs["vocab_size"] = config_gpt["n_vocab"]
    configs["context_size"] = config_gpt["n_ctx"]
    configs["embedding_size"] = config_gpt["n_embd"]
    hidden_dim = config_gpt["n_embd"]
    configs["embed"] = {
        "dim": hidden_dim,
    }
    configs["position_size"] = config_gpt["n_ctx"]
    configs["pos_embed"] = {"dim": hidden_dim}
    configs["decoder"] = {
        "dim": hidden_dim,
        "num_blocks": config_gpt["n_layer"],
        "multihead_attention": {
            "use_bias": True,
            "num_units": hidden_dim,
            "num_heads": config_gpt["n_head"],
            "output_dim": hidden_dim,
        },
        "initializer": {
            "type": "variance_scaling_initializer",
            "kwargs": {
                "scale": 1.0,
                "mode": "fan_avg",
                "distribution": "uniform",
            },
        },
        "poswise_feedforward": {
            "layers": [{
                "type": "Dense",
                "kwargs": {
                    "name": "conv1",
                    "units": hidden_dim * 4,
                    "activation": "gelu",
                    "use_bias": True,
                }
            }, {
                "type": "Dense",
                "kwargs": {
                    "name": "conv2",
                    "units": hidden_dim,
                    "use_bias": True,
                }
            }],
            "name":
            "ffn",
        },
    }
    return HParams(configs, default_hparams=None)
Beispiel #5
0
    def __init__(self, data_hparams, hparams=None):
        ModelBase.__init__(self, hparams)

        self._data_hparams = HParams(data_hparams,
                                     PairedTextData.default_hparams())

        self._src_vocab = None
        self._tgt_vocab = None
        self._src_embedder = None
        self._tgt_embedder = None
        self._connector = None
        self._encoder = None
        self._decoder = None
Beispiel #6
0
 def __init__(self, hparams=None):
     self._hparams = HParams(hparams,
                             self.default_hparams(),
                             allow_new_hparam=True)