def transformer_base_single_gpu_mjc():
hparams = transformer.transformer_base_single_gpu()
# hparams.eval_freq_in_steps = 100 # (2000 in default)
# hparams.iterations_per_loop = 100 # (100 in default)
hparams = register_self_defined_params(hparams)
hparams.batch_size = 512
return hparams
def word2ner_conll_hparams():
hparams = transformer.transformer_base_single_gpu()
hparams.learning_rate_warmup_steps = 4000
hparams.num_hidden_layers = 4
hparams.learning_rate = 0.1
hparams.hidden_size = 256
return hparams
def transformer_base_single_gpu_shs_train():
hparams = transformer.transformer_base_single_gpu()
hparams = register_self_defined_params(hparams)
hparams.shs_regularization = True
hparams.shs_finetune = False
hparams.print_sparsity = True
return hparams
def transformer_zhen():
hparams = transformer.transformer_base_single_gpu()
hparams.num_hidden_layers = 2
hparams.hidden_size = 128
hparams.filter_size = 512
hparams.attention_dropout = 0.6
return hparams
def word2ner_subword_hparams():
hparams = transformer.transformer_base_single_gpu()
hparams.learning_rate_warmup_steps = 6000
hparams.num_hidden_layers = 4
hparams.learning_rate = 0.1
hparams.hidden_size = 128
#hparams.optimizer = 'Adadelta'
return hparams
def word2ner_msra_hparams():
hparams = transformer.transformer_base_single_gpu()
hparams.learning_rate_warmup_steps = 6000
hparams.num_hidden_layers = 4
hparams.learning_rate = 0.1
hparams.hidden_size = 128
#hparams.clip_grad_norm = 5.0
return hparams
def transformer_base_single_gpu_ssl_finetune():
hparams = transformer.transformer_base_single_gpu()
hparams = register_self_defined_params(hparams)
# duing the finetune step, the ssl regularition term will be turned off
hparams.ssl_regularization = False
hparams.ssl_finetune = True
# hparams.learning_rate = hparams.learning_rate/10
hparams.learning_rate_constant = hparams.learning_rate_constant / 10
return hparams
def word2ner_hparams_long():
hparams = transformer.transformer_base_single_gpu()
hparams.learning_rate_warmup_steps = 16000
hparams.num_hidden_layers = 4
hparams.learning_rate = 0.03
hparams.hidden_size = 128
hparams.filter_size = 1024
hparams.max_length = 512
return hparams
def librispeech_hparams():
hparams = transformer.transformer_base_single_gpu(
) # Or whatever you'd like to build off.
hparams.batch_size = 36
hparams.audio_compression = 8
hparams.hidden_size = 2048
hparams.max_input_seq_length = 600000
hparams.max_target_seq_length = 350
hparams.max_length = hparams.max_input_seq_length
hparams.min_length_bucket = hparams.max_input_seq_length // 2
hparams.learning_rate = 0.05
hparams.train_steps = 5000000
hparams.num_hidden_layers = 4
return hparams
def enkrch_hparams():
hparams = transformer.transformer_base_single_gpu(
) # Or whatever you'd like to build off.
hparams.learning_rate = 0.04
return hparams
def transformer_yelp_sentiment():
# https://github.com/tensorflow/tensor2tensor/blob/99750c4b6858de46b75b067e3a967fe74da1c874/tensor2tensor/models/transformer.py#L1040
hparams = transformer.transformer_base_single_gpu()
return hparams
def text_simplification_hparams(self):
hparams = transformer.transformer_base_single_gpu()
hparams.batch_size = 1024
return hparams
def transformer_base_small_gpu():
"""HParams for transformer base model for single GPU with low memory."""
hparams = transformer_base_single_gpu()
hparams.batch_size = 512
hparams.learning_rate_warmup_steps = 16000
return hparams