def get_model_config():
default_config = openai_transformer_config()
config = AttrDict({
'bpe_vocab_path': 'parameters/bpe.vocab',
'bpe_codes_path': 'parameters/bpe.code',
'checkpoint_path': 'checkpoints/last_checkpoint',
'n_layers': default_config.n_layers,
'n_pos_embeddings': 512,
'embeddings_size': default_config.embeddings_size,
'n_heads': default_config.n_heads,
'dropout': default_config.dropout,
'embed_dropout': default_config.embed_dropout,
'attn_dropout': default_config.attn_dropout,
'ff_dropout': default_config.ff_dropout,
'max_seq_len': 256,
'beam_size': 3,
'diversity_coef': 0,
'diversity_groups': 1,
'annealing_topk': None,
'annealing': 0,
'length_penalty': 0.6,
'n_segments': None
})
return config
def get_model_config_dialog():
default_config = openai_transformer_config()
config = AttrDict({
'vocab_path':
'/root/generation_with_augmentation/parameters/vocab.txt',
'checkpoint_path':
'/root/generation_with_augmentation/checkpoints/dialog_300k/crowded_extend_both_kd/last_checkpoint',
'n_layers': 12,
'n_pos_embeddings': 512,
'embeddings_size': default_config.embeddings_size,
'n_heads': default_config.n_heads,
'dropout': default_config.dropout,
'embed_dropout': default_config.embed_dropout,
'attn_dropout': default_config.attn_dropout,
'ff_dropout': default_config.ff_dropout,
'max_seq_len': 32,
'beam_size': 1,
'diversity_coef': 0,
'diversity_groups': 1,
'temperature': 1.0,
'annealing_topk': None,
'annealing': 0,
'length_penalty': 1.0,
'n_segments': None
})
return config
def get_model_config(args):
default_config = openai_transformer_config()
config = AttrDict({
'bpe_vocab_path': './parameters/bpe.vocab',
'bpe_codes_path': './parameters/bpe.code',
'checkpoint_path':
'./checkpoints/last_checkpoint', # Keep the checpoint folder for the checkpoints of the agents
'n_layers': default_config.n_layers,
'n_pos_embeddings': 512,
'embeddings_size': default_config.embeddings_size,
'n_heads': default_config.n_heads,
'dropout': default_config.dropout,
'embed_dropout': default_config.embed_dropout,
'attn_dropout': default_config.attn_dropout,
'ff_dropout': default_config.ff_dropout,
'normalize_embeddings': args.normalize_embeddings,
'max_seq_len': 128,
'beam_size': args.beam_size,
'diversity_coef': args.diversity_coef,
'diversity_groups': args.diversity_groups,
'annealing_topk': args.annealing_topk,
'annealing': args.annealing,
'length_penalty': args.length_penalty,
'n_segments': None,
'constant_embedding': args.constant_embedding,
'multiple_choice_head': args.multiple_choice_head,
'share_models': True,
'successive_attention': args.successive_attention,
'sparse_embeddings': args.sparse_embeddings,
'shared_attention': args.shared_attention,
'dialog_embeddings': args.dialog_embeddings,
'single_input': args.single_input,
'use_start_end': args.use_start_end,
'apex_level': args.apex_level, # 'O0', 'O1', 'O2', 'O3',
'bs_temperature': args.bs_temperature,
'bs_nucleus_p': args.bs_nucleus_p,
'same_embedding_lm': args.same_embedding_lm,
})
return config
def get_model_config():
default_config = openai_transformer_config()
config = AttrDict({
'bpe_vocab_path':
'./parameters/bpe.vocab',
'bpe_codes_path':
'./parameters/bpe.code',
'checkpoint_path':
'./checkpoints/last_checkpoint', # Keep the checpoint folder for the checkpoints of the agents
'n_layers':
default_config.n_layers,
'n_pos_embeddings':
512,
'embeddings_size':
default_config.embeddings_size,
'n_heads':
default_config.n_heads,
'dropout':
default_config.dropout,
'embed_dropout':
default_config.embed_dropout,
'attn_dropout':
default_config.attn_dropout,
'ff_dropout':
default_config.ff_dropout,
'normalize_embeddings':
env_config('NORMALIZE_EMBEDDINGS', default=False, cast=bool),
'max_seq_len':
128,
'beam_size':
env_config('BEAM_SIZE', default=3, cast=int),
'diversity_coef':
env_config('DIVERSITY_COEF', default=0, cast=int),
'diversity_groups':
env_config('DIVERSITY_GROUP', default=1, cast=int),
'annealing_topk':
env_config('ANNEALING_TOPK', default=None, cast=cast2(int)),
'annealing':
env_config('ANNEALING', default=0, cast=float),
'length_penalty':
env_config('LENGTH_PENALTY', default=0.6, cast=float),
'n_segments':
None,
'constant_embedding':
env_config('CONSTANT_EMBEDDINGS', default=False, cast=bool),
'multiple_choice_head':
env_config('MULTIPLE_CHOICE_HEAD', default=False, cast=bool),
'share_models':
env_config('SHARE_MODELS', default=True, cast=bool),
'successive_attention':
env_config('SUCCESSIVE_ATTENTION', default=False, cast=bool),
'sparse_embeddings':
env_config('SPARSE_EMBEDDINGS', default=True, cast=bool),
'shared_attention':
env_config('SHARED_ATTENTION', default=True, cast=bool),
'dialog_embeddings':
env_config('DIALOG_EMBEDDINGS', default=True, cast=bool),
'single_input':
env_config('SINGLE_INPUT', default=False, cast=bool),
'use_start_end':
env_config('USE_START_END', default=False, cast=bool),
'apex_level':
env_config('APEX_LEVEL', default=None,
cast=cast2(str)), # 'O0', 'O1', 'O2', 'O3',
'bs_temperature':
env_config('BS_TEMPERATURE', default=1, cast=float),
'bs_nucleus_p':
env_config('BS_NUCLEUS_P', default=0, cast=float)
})
return config