def get_config():
"""Get the default hyperparameter configuration."""
config = base_match_config.get_config()
config.model_type = "transformer"
config.attention_fn = favor.make_fast_softmax_attention(
qkv_dim=config.qkv_dim // config.num_heads, lax_scan_unroll=16)
return config
def get_config():
"""Get the default hyperparameter configuration."""
config = base_match_config.get_config()
config.random_seed = 2
config.model_type = "transformer"
num_realizations = 64
config.attention_fn = favor.make_fast_generalized_attention(
qkv_dim=num_realizations,
features_type='deterministic',
kernel_fn=jax.nn.relu,
lax_scan_unroll=16)
config.model_kwargs = dict(add_pos_emb=False,
qk_transform_fn_factory=functools.partial(
make_spe_transform_fn,
spe_cls=spe.ConvSPE,
spe_kwargs=dict(
num_realizations=num_realizations,
kernel_size=128),
shared=True))
config.batch_size = 8
config.learning_rate = 0.005
config.num_train_steps = 15000
config.warmup = 3000
config.eval_frequency = 1500
return config
def get_config():
"""Get the default hyperparameter configuration."""
config = base_match_config.get_config()
config.random_seed = 2
config.model_type = "transformer"
num_realizations = 64
config.model_kwargs = dict(
add_pos_emb=False,
qk_transform_fn_factory=functools.partial(
make_spe_transform_fn,
spe_cls=spe.SineSPE,
spe_kwargs=dict(
num_realizations=num_realizations,
num_sines=10
),
)
)
config.attention_fn = favor.make_fast_softmax_attention(
qkv_dim=num_realizations,
lax_scan_unroll=16)
config.batch_size = 8
config.learning_rate = 0.005
config.num_train_steps = 15000
config.warmup = 3000
config.eval_frequency = 1500
return config
def get_config():
"""Get the default hyperparameter configuration."""
config = base_match_config.get_config()
config.model_type = "transformer"
config.attention_fn = favor.make_fast_generalized_attention(
qkv_dim=config.qkv_dim // config.num_heads,
features_type='ortho',
kernel_fn=jax.nn.relu,
lax_scan_unroll=16)
return config
def get_config():
"""Get the default hyperparameter configuration."""
config = base_match_config.get_config()
config.random_seed = 2
config.model_type = "transformer"
config.attention_fn = favor.make_fast_softmax_attention(
qkv_dim=config.qkv_dim // config.num_heads, lax_scan_unroll=16)
config.batch_size = 8
config.learning_rate = 0.005
config.num_train_steps = 15000
config.warmup = 3000
config.eval_frequency = 1500
return config
def get_config():
"""Get the default hyperparameter configuration."""
config = base_match_config.get_config()
config.random_seed = 0
config.model_type = "transformer"
config.attention_fn = favor.make_fast_generalized_attention(
qkv_dim=config.qkv_dim // config.num_heads,
features_type='deterministic',
kernel_fn=jax.nn.relu,
lax_scan_unroll=16)
config.batch_size = 8
config.learning_rate = 0.005
config.num_train_steps = 15000
config.warmup = 3000
config.eval_frequency = 1500
return config
def get_config():
"""Get the default hyperparameter configuration."""
config = base_match_config.get_config()
config.model_type = "transformer"
return config
