class AdaptiveLossConfig(FairseqDataclass):
sentence_avg: bool = II("optimization.sentence_avg")
ddp_backend: DDP_BACKEND_CHOICES = II("distributed_training.ddp_backend")
class SpeechRecognitionHybridConfig(FairseqDataclass):
data: Optional[str] = field(default=None,
metadata={"help": "path to data directory"})
dict: Optional[str] = field(default=None,
metadata={"help": "path to the dictionary"})
non_lang_syms: Optional[str] = field(
default=None,
metadata={
"help":
"path to a file listing non-linguistic symbols, e.g., <NOISE> "
"etc. One entry per line. To be filtered out when calculating WER/CER"
},
)
wer_output_filter: Optional[str] = field(
default=None,
metadata={"help": "path to wer_output_filter file for WER evaluation"},
)
max_source_positions: Optional[int] = field(
default=1024,
metadata={"help": "max number of tokens in the source sequence"})
max_target_positions: Optional[int] = field(
default=1024,
metadata={"help": "max number of tokens in the target sequence"})
upsample_primary: int = field(
default=1,
metadata={"help": "amount to upsample primary dataset"},
)
num_batch_buckets: Optional[int] = field(
default=0,
metadata={
"help":
"if >0, then bucket source and target lengths into N "
"buckets and pad accordingly; this is useful on TPUs "
"to minimize the number of compilations"
},
)
feat_in_channels: int = field(default=1,
metadata={"help": "feature input channels"})
global_cmvn_stats_path: Optional[str] = field(
default=None,
metadata={
"help":
"If not None, apply global cmvn using this global cmvn stats file (.npz)."
},
)
specaugment_config: Optional[str] = field(
default=None,
metadata={
"help":
"SpecAugment config string. If not None and not empty, "
"then apply SpecAugment. Should be an evaluatable expression of "
"a python dict. See speech_tools.specaug_interpolate.specaug() for "
"all allowed arguments. Argments not appearing in this string "
"will take on their default values"
},
)
num_targets: int = field(
default=3000,
metadata={
"help": "number of targets for training (e.g., num pdf-ids)"
},
)
initial_state_prior_file: Optional[str] = field(
default=None,
metadata={
"help":
"path to the file containing initial state prior. Only relevant "
"with cross-entropy training"
},
)
state_prior_update_interval: Optional[int] = field(
default=None,
metadata={
"help":
"state prior estimate will be updated every this number of updates "
"during training. If None, then use the initial value estimated from the "
"alignments. Only relevant with cross-entropy training"
},
)
state_prior_update_smoothing: Optional[float] = field(
default=0.1,
metadata={
"help":
"smoothing factor while updating state prior estimate. Only "
"relevant with cross-entropy training"
},
)
chunk_width: Optional[int] = field(
default=None,
metadata={
"help":
"chunk width for train/test data. Only relevant with chunk-wise "
"training (including both cross-entropy and Lattice-free MMI). "
"Do utterance-wise training/test if not specified"
},
)
chunk_left_context: Optional[int] = field(
default=0,
metadata={"help": "number of frames appended to the left of a chunk"},
)
chunk_right_context: Optional[int] = field(
default=0,
metadata={"help": "number of frames appended to the right of a chunk"},
)
label_delay: Optional[int] = field(
default=0,
metadata={
"help":
"offet of alignments as prediction labels. Maybe useful "
"in archs such as asymmetric convolution, unidirectional LSTM, etc. "
"It can be negative. Only relevant with chunk-wise cross-entropy training"
},
)
# TODO common vars below add to parent
seed: int = II("common.seed")
data_buffer_size: int = II("dataset.data_buffer_size")
tpu: bool = II("common.tpu")
train_subset: str = II("dataset.train_subset")
valid_subset: str = II("dataset.valid_subset")
gen_subset: str = II("dataset.gen_subset")
required_seq_len_multiple: int = II("dataset.required_seq_len_multiple")
criterion_name: str = II("criterion._name")
max_epoch: int = II(
"optimization.max_epoch") # to determine whether in trainig stage
class ModelConfig:
image_size: Tuple[int, int] = II('preprocess.target_size')
drop_rate: float = 0.5
class SpeechTransformerTransducerConfig(FairseqDataclass):
activation_fn: ChoiceEnum(utils.get_available_activation_fns()) = field(
default="relu",
metadata={"help": "activation function to use"},
)
dropout: float = field(default=0.1,
metadata={"help": "dropout probability"})
attention_dropout: float = field(
default=0.0,
metadata={"help": "dropout probability for attention weights"})
activation_dropout: float = field(
default=0.0,
metadata={
"help": "dropout probability after activation in FFN.",
"alias": "--relu-dropout",
},
)
adaptive_input: bool = field(
default=False,
metadata={"help": "if set, uses adaptive input"},
)
encoder: SpeechEncoderConfig = SpeechEncoderConfig()
decoder: SpeechDecoderConfig = SpeechDecoderConfig()
joint_dim: int = field(default=512,
metadata={"help": "joint network output dimension"})
share_decoder_input_output_embed: bool = field(
default=False,
metadata={"help": "share decoder input and output embeddings"})
no_token_positional_embeddings: bool = field(
default=False,
metadata={
"help":
"if True, disables positional embeddings (outside self attention)"
},
)
layernorm_embedding: bool = field(
default=True, metadata={"help": "add layernorm to embedding"})
no_scale_embedding: bool = field(
default=False, metadata={"help": "if True, dont scale embeddings"})
checkpoint_activations: bool = field(
default=False,
metadata={
"help":
"checkpoint activations at each layer, which saves GPU memory usage at the cost of some additional compute"
},
)
offload_activations: bool = field(
default=False,
metadata={
"help":
"checkpoint activations at each layer, then save to gpu. Sets --checkpoint-activations."
},
)
# args for Training with Quantization Noise for Extreme Model Compression ({Fan*, Stock*} et al., 2020)
quant_noise: QuantNoiseConfig = field(default=QuantNoiseConfig())
min_params_to_wrap: int = field(
default=DEFAULT_MIN_PARAMS_TO_WRAP,
metadata={
"help":
"minimum number of params for a layer to be wrapped with FSDP() when "
"training with --ddp-backend=fully_sharded. Smaller values will "
"improve memory efficiency, but may make torch.distributed "
"communication less efficient due to smaller input sizes. This option "
"is set to 0 (i.e., always wrap) when --checkpoint-activations or "
"--offload-activations are passed."
},
)
export: bool = field(
default=False,
metadata={"help": "make the layernorm exportable with torchscript."},
)
# options from other parts of the config
max_source_positions: Optional[int] = II("task.max_source_positions")
max_target_positions: Optional[int] = II("task.max_target_positions")
tpu: bool = II("common.tpu")
# We need to make this hierarchical dataclass like the flat namespace
# __getattr__ and __setattr__ here allow backward compatibility
# for subclasses of Transformer(Legacy) that depend on read/write on
# the flat namespace.
def __getattr__(self, name):
match = re.match(_NAME_PARSER, name)
if match:
sub = safe_getattr(self, match[1])
return safe_getattr(sub, match[2])
raise AttributeError(f"invalid argument {name}.")
def __setattr__(self, name, value):
match = re.match(_NAME_PARSER, name)
if match:
sub = safe_getattr(self, match[1])
setattr(sub, match[2], value)
else:
super().__setattr__(name, value)
@staticmethod
def _copy_keys(args, cls, prefix, seen):
return TransformerConfig._copy_keys(args, cls, prefix, seen)
@classmethod
def from_namespace(cls, args):
if args is None:
return None
if not isinstance(args, cls):
seen = set()
config = cls()
# currently, we can go generically from DC fields to args hierarchically
# but we can't easily deconstruct a flat namespace to a hierarchical
# DC. Mostly because we could have a sub-dc called `decoder-foo` that should not
# go to the sub struct called `decoder`. There are ways to go around this, but let's keep it simple
# for now.
for fld in fields(cls):
# concretelly, the transformer_config know what sub-dc it has, so we go through all the dc fields
# and if it's one that has a sub-dc, we build that sub-dc with `copy_keys()`
if fld.name == "decoder":
if safe_hasattr(args, "decoder"):
# in some cases, the args we receive is already structured (as DictConfigs), so let's just build the correct DC
seen.add("decoder")
config.decoder = SpeechDecoderConfig(**args.decoder)
else:
config.decoder = cls._copy_keys(
args, SpeechDecoderConfig, "decoder", seen)
elif fld.name == "encoder":
# same but for encoder
if safe_hasattr(args, "encoder"):
seen.add("encoder")
config.encoder = SpeechEncoderConfig(**args.encoder)
else:
config.encoder = cls._copy_keys(
args, SpeechEncoderConfig, "encoder", seen)
elif fld.name == "quant_noise":
# same but for quant_noise
if safe_hasattr(args, "quant_noise"):
seen.add("quant_noise")
config.quant_noise = QuantNoiseConfig(
**args.quant_noise)
else:
config.quant_noise = cls._copy_keys(
args, QuantNoiseConfig, "quant_noise", seen)
elif safe_hasattr(args, fld.name):
# if it's not a structure field, it's just a normal field, copy it over
seen.add(fld.name)
setattr(config, fld.name, safe_getattr(args, fld.name))
# we got all the fields defined in the dataclass, but
# the argparse namespace might have extra args for two reasons:
# - we are in a legacy class so all the args are not declared in the dataclass. Ideally once everyone has defined a dataclass for their model, we won't need this
# - some places expect args to be there but never define them
args_dict = (args._asdict() if safe_hasattr(args, "_asdict") else
vars(args) if safe_hasattr(args, "__dict__") else {}
) # namedtupled doesn't have __dict__ :-/
for key, value in args_dict.items():
if key not in seen:
setattr(config, key, value)
return config
else:
return args
class TransformerLanguageModelConfig(FairseqDataclass):
activation_fn: ChoiceEnum(utils.get_available_activation_fns()) = field(
default="relu", metadata={"help": "activation function to use"}
)
dropout: float = field(default=0.1, metadata={"help": "dropout probability"})
attention_dropout: float = field(
default=0.0, metadata={"help": "dropout probability for attention weights"}
)
activation_dropout: float = field(
default=0.0, metadata={"help": "dropout probability after activation in FFN."}
)
relu_dropout: float = field(
default=0.0, metadata={"help": "dropout probability after activation in FFN."}
)
decoder_embed_dim: int = field(
default=512, metadata={"help": "decoder embedding dimension"}
)
decoder_output_dim: int = field(
default=512, metadata={"help": "decoder output dimension"}
)
decoder_input_dim: int = field(
default=512, metadata={"help": "decoder input dimension"}
)
decoder_ffn_embed_dim: int = field(
default=2048, metadata={"help": "decoder embedding dimension for FFN"}
)
decoder_layers: int = field(default=6, metadata={"help": "num decoder layers"})
decoder_attention_heads: int = field(
default=8, metadata={"help": "num decoder attention heads"}
)
decoder_normalize_before: bool = field(
default=False, metadata={"help": "apply layernorm before each decoder block"}
)
no_decoder_final_norm: bool = field(
default=False,
metadata={"help": "don't add an extra layernorm after the last decoder block"},
)
adaptive_softmax_cutoff: Optional[str] = field(
default=None,
metadata={
"help": "comma separated list of adaptive softmax cutoff points. "
"Must be used with adaptive_loss criterion"
},
)
adaptive_softmax_dropout: float = field(
default=0,
metadata={"help": "sets adaptive softmax dropout for the tail projections"},
)
adaptive_softmax_factor: float = field(
default=4, metadata={"help": "adaptive input factor"}
)
no_token_positional_embeddings: bool = field(
default=False,
metadata={
"help": "if set, disables positional embeddings (outside self attention)"
},
)
share_decoder_input_output_embed: bool = field(
default=False, metadata={"help": "share decoder input and output embeddings"}
)
character_embeddings: bool = field(
default=False,
metadata={
"help": "if set, uses character embedding convolutions to produce token embeddings"
},
)
character_filters: str = field(
default="[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]",
metadata={"help": "size of character embeddings"},
)
character_embedding_dim: int = field(
default=4, metadata={"help": "size of character embeddings"}
)
char_embedder_highway_layers: int = field(
default=2,
metadata={"help": "number of highway layers for character token embeddder"},
)
adaptive_input: bool = field(
default=False, metadata={"help": "if set, uses adaptive input"}
)
adaptive_input_factor: float = field(
default=4, metadata={"help": "adaptive input factor"}
)
adaptive_input_cutoff: Optional[str] = field(
default=None,
metadata={"help": "comma separated list of adaptive input cutoff points."},
)
tie_adaptive_weights: bool = field(
default=False,
metadata={
"help": "if set, ties the weights of adaptive softmax and adaptive input"
},
)
tie_adaptive_proj: bool = field(
default=False,
metadata={
"help": "if set, ties the projection weights of adaptive softmax and adaptive input"
},
)
decoder_learned_pos: bool = field(
default=False,
metadata={"help": "use learned positional embeddings in the decoder"},
)
layernorm_embedding: bool = field(
default=False, metadata={"help": "add layernorm to embedding"}
)
no_scale_embedding: bool = field(
default=False, metadata={"help": "if True, dont scale embeddings"}
)
checkpoint_activations: bool = field(
default=False, metadata={"help": "checkpoint activations at each layer"}
)
offload_activations: bool = field(
default=False,
metadata={"help": "move checkpointed activations to CPU after they are used."},
)
# config for "Reducing Transformer Depth on Demand with Structured Dropout" (Fan et al., 2019)
decoder_layerdrop: float = field(
default=0.0, metadata={"help": "LayerDrop probability for decoder"}
)
decoder_layers_to_keep: Optional[str] = field(
default=None,
metadata={
"help": "which layers to *keep* when pruning as a comma-separated list"
},
)
# config for Training with Quantization Noise for Extreme Model Compression ({Fan*, Stock*} et al., 2020)
quant_noise_pq: float = field(
default=0.0,
metadata={"help": "iterative PQ quantization noise at training time"},
)
quant_noise_pq_block_size: int = field(
default=8,
metadata={"help": "block size of quantization noise at training time"},
)
quant_noise_scalar: float = field(
default=0.0,
metadata={
"help": "scalar quantization noise and scalar quantization at training time"
},
)
# config for Fully Sharded Data Parallel (FSDP) training
min_params_to_wrap: int = field(
default=DEFAULT_MIN_PARAMS_TO_WRAP,
metadata={
"help": (
"minimum number of params for a layer to be wrapped with FSDP() when "
"training with --ddp-backend=fully_sharded. Smaller values will "
"improve memory efficiency, but may make torch.distributed "
"communication less efficient due to smaller input sizes. This option "
"is set to 0 (i.e., always wrap) when --checkpoint-activations or "
"--offload-activations are passed."
)
},
)
# config for "BASE Layers: Simplifying Training of Large, Sparse Models"
base_layers: Optional[int] = field(
default=0, metadata={"help": "number of BASE layers in total"}
)
base_sublayers: Optional[int] = field(
default=1, metadata={"help": "number of sublayers in each BASE layer"}
)
base_shuffle: Optional[int] = field(
default=1,
metadata={"help": "shuffle tokens between workers before computing assignment"},
)
# NormFormer
scale_fc: Optional[bool] = field(
default=False,
metadata={"help": "Insert LayerNorm between fully connected layers"},
)
scale_attn: Optional[bool] = field(
default=False, metadata={"help": "Insert LayerNorm after attention"}
)
scale_heads: Optional[bool] = field(
default=False,
metadata={"help": "Learn a scale coefficient for each attention head"},
)
scale_resids: Optional[bool] = field(
default=False,
metadata={"help": "Learn a scale coefficient for each residual connection"},
)
# options from other parts of the config
add_bos_token: bool = II("task.add_bos_token")
tokens_per_sample: int = II("task.tokens_per_sample")
max_target_positions: Optional[int] = II("task.max_target_positions")
tpu: bool = II("common.tpu")
class AudioPretrainingConfig(FairseqDataclass):
data: str = field(default=MISSING, metadata={"help": "path to data directory"})
augment_data: Optional[str] = field(
default=None,
metadata={"help": "path to noise/rir data directory"},
)
labels: Optional[str] = field(
default=None,
metadata={"help": "extension of the label file to load, used for fine-tuning"},
)
binarized_dataset: bool = field(
default=False,
metadata={
"help": "if true, loads binarized dataset (useful for very large datasets). "
"See examples/wav2vec/scripts/binarize_manifest.sh"
},
)
sample_rate: int = field(
default=16_000,
metadata={
"help": "target sample rate. audio files will be up/down sampled to this rate"
},
)
normalize: bool = field(
default=False,
metadata={"help": "if set, normalizes input to have 0 mean and unit variance"},
)
enable_padding: bool = field(
default=False, metadata={"help": "pad shorter samples instead of cropping"}
)
max_sample_size: Optional[int] = field(
default=None, metadata={"help": "max sample size to crop to for batching"}
)
min_sample_size: Optional[int] = field(
default=None, metadata={"help": "min sample size to skip small examples"}
)
# Options for reporting WER metrics during validation. Only applicable to
# Seq2Seq models during fine-tuning
eval_wer: bool = field(
default=False, metadata={"help": "compute WER for Seq2Seq models"}
)
eval_wer_config: GenerationConfig = field(
default_factory=lambda: GenerationConfig(),
metadata={"help": "beam search config for evaluating wer during training"},
)
eval_wer_tokenizer: Any = field(
default=None,
metadata={"help": "tokenizer config for evaluating wer during training"},
)
eval_wer_post_process: str = field(
default="letter",
metadata={
"help": "remove BPE tokens before scoring (can be sentencepiece, letter, and more)"
},
)
autoregressive: bool = field(
default=False,
metadata={
"help": "required for autoregressive decoders (like seq2seq models); "
"adds 'prev_output_tokens' to input and appends eos to target"
},
)
num_batch_buckets: int = field(
default=0,
metadata={"help": "number of buckets"},
)
precompute_mask_indices: bool = field(
default=False,
metadata={
"help": "flag to compute mask indices in data preparation.",
},
)
inferred_w2v_config: Optional[InferredW2vConfig] = field(
default=None,
metadata={
"help": "wav2vec 2.0 masking arguments used to pre-compute masks (required for TPU)",
},
)
tpu: bool = II("common.tpu")
class DatasetConfig(FairseqDataclass):
num_workers: int = field(
default=1,
metadata={"help": "how many subprocesses to use for data loading"})
skip_invalid_size_inputs_valid_test: bool = field(
default=False,
metadata={
"help": "ignore too long or too short lines in valid and test set"
},
)
max_tokens: Optional[int] = field(
default=None, metadata={"help": "maximum number of tokens in a batch"})
batch_size: Optional[int] = field(
default=None,
metadata={
"help": "number of examples in a batch",
"argparse_alias": "--max-sentences",
},
)
required_batch_size_multiple: int = field(
default=8,
metadata={"help": "batch size will be a multiplier of this value"})
required_seq_len_multiple: int = field(
default=1,
metadata={
"help":
"maximum sequence length in batch will be a multiplier of this value"
},
)
dataset_impl: Optional[DATASET_IMPL_CHOICES] = field(
default=None, metadata={"help": "output dataset implementation"})
data_buffer_size: int = field(
default=10, metadata={"help": "Number of batches to preload"})
train_subset: str = field(
default="train",
metadata={
"help": "data subset to use for training (e.g. train, valid, test)"
},
)
valid_subset: str = field(
default="valid",
metadata={
"help":
"comma separated list of data subsets to use for validation"
" (e.g. train, valid, test)"
},
)
validate_interval: int = field(
default=1, metadata={"help": "validate every N epochs"})
validate_interval_updates: int = field(
default=0, metadata={"help": "validate every N updates"})
validate_after_updates: int = field(
default=0,
metadata={"help": "dont validate until reaching this many updates"})
fixed_validation_seed: Optional[int] = field(
default=None,
metadata={"help": "specified random seed for validation"})
disable_validation: bool = field(default=False,
metadata={"help": "disable validation"})
max_tokens_valid: Optional[int] = field(
default=II("dataset.max_tokens"),
metadata={
"help":
"maximum number of tokens in a validation batch"
" (defaults to --max-tokens)"
},
)
batch_size_valid: Optional[int] = field(
default=II("dataset.batch_size"),
metadata={
"help":
"batch size of the validation batch (defaults to --batch-size)",
"argparse_alias": "--max-sentences-valid",
},
)
max_valid_steps: Optional[int] = field(default=None,
metadata={
'help':
'How many batches to evaluate',
"argparse_alias": "--nval"
})
curriculum: int = field(
default=0,
metadata={"help": "don't shuffle batches for first N epochs"})
gen_subset: str = field(
default="test",
metadata={"help": "data subset to generate (train, valid, test)"},
)
num_shards: int = field(
default=1, metadata={"help": "shard generation over N shards"})
shard_id: int = field(
default=0,
metadata={"help": "id of the shard to generate (id < num_shards)"})
class InterpolationList:
list: List[float] = II("optimization.lr")
class AudioPretrainingConfig(FairseqDataclass):
data: str = field(default=MISSING,
metadata={"help": "path to data directory"})
labels: Optional[str] = field(
default=None,
metadata={
"help": "extension of the label file to load, used for fine-tuning"
},
)
binarized_dataset: bool = field(
default=False,
metadata={
"help":
"if true, loads binarized dataset (useful for very large datasets). "
"See examples/wav2vec/scripts/binarize_manifest.sh"
},
)
sample_rate: int = field(
default=16_000,
metadata={
"help":
"target sample rate. audio files will be up/down sampled to this rate"
},
)
normalize: bool = field(
default=False,
metadata={
"help": "if set, normalizes input to have 0 mean and unit variance"
},
)
enable_padding: bool = field(
default=False,
metadata={"help": "pad shorter samples instead of cropping"})
max_sample_size: Optional[int] = field(
default=None,
metadata={"help": "max sample size to crop to for batching"})
min_sample_size: Optional[int] = field(
default=None,
metadata={"help": "min sample size to skip small examples"})
# Options for reporting WER metrics during validation. Only applicable to
# Seq2Seq models during fine-tuning
eval_wer: bool = field(default=False,
metadata={"help": "compute WER for Seq2Seq models"})
eval_wer_config: GenerationConfig = field(
default_factory=lambda: GenerationConfig(),
metadata={
"help": "beam search config for evaluating wer during training"
},
)
eval_wer_tokenizer: Any = field(
default=None,
metadata={
"help": "tokenizer config for evaluating wer during training"
},
)
eval_wer_post_process: str = field(
default="letter",
metadata={
"help":
"remove BPE tokens before scoring (can be sentencepiece, letter, and more)"
},
)
autoregressive: bool = field(
default=False,
metadata={
"help":
"required for autoregressive decoders (like seq2seq models); "
"adds 'prev_output_tokens' to input and appends eos to target"
},
)
num_batch_buckets: int = field(
default=0,
metadata={"help": "number of buckets"},
)
precompute_mask_indices: bool = field(
default=False,
metadata={
"help": "flag to compute mask indices in data preparation.",
},
)
# The following are needed to precompute mask and mask channel indices
# before model's forward.
mask_length: Optional[int] = II("model.mask_length")
mask_prob: Optional[float] = II("model.mask_prob")
mask_selection: Optional[str] = II("model.mask_selection")
mask_other: Optional[float] = II("model.mask_other")
no_mask_overlap: Optional[bool] = II("model.no_mask_overlap")
mask_min_space: Optional[int] = II("model.mask_min_space")
mask_channel_length: Optional[int] = II("model.mask_channel_length")
mask_channel_prob: Optional[float] = II("model.mask_channel_prob")
mask_channel_selection: Optional[str] = II("model.mask_channel_selection")
mask_channel_other: Optional[float] = II("model.mask_channel_other")
no_mask_channel_overlap: Optional[bool] = II(
"model.no_mask_channel_overlap")
mask_channel_min_space: Optional[int] = II("model.mask_channel_min_space")
conv_feature_layers: Optional[str] = II("model.conv_feature_layers")
encoder_embed_dim: Optional[int] = II("model.encoder_embed_dim")
tpu: bool = II("common.tpu")
class TranslationDAConfig(FairseqDataclass):
data: Optional[str] = field(
default=None,
metadata={
"help":
"colon separated path to data directories list, will be iterated upon during epochs "
"in round-robin manner; however, valid and test data are always in the first directory "
"to avoid the need for repeating them in all directories"
},
)
source_lang: Optional[str] = field(
default=None,
metadata={
"help": "source language",
"argparse_alias": "-s",
},
)
target_lang: Optional[str] = field(
default=None,
metadata={
"help": "target language",
"argparse_alias": "-t",
},
)
load_alignments: bool = field(
default=False, metadata={"help": "load the binarized alignments"})
left_pad_source: bool = field(
default=True, metadata={"help": "pad the source on the left"})
left_pad_target: bool = field(
default=False, metadata={"help": "pad the target on the left"})
max_source_positions: int = field(
default=1024,
metadata={"help": "max number of tokens in the source sequence"})
max_target_positions: int = field(
default=1024,
metadata={"help": "max number of tokens in the target sequence"})
upsample_primary: int = field(
default=-1,
metadata={"help": "the amount of upsample primary dataset"})
truncate_source: bool = field(
default=False,
metadata={"help": "truncate source to max-source-positions"})
num_batch_buckets: int = field(
default=0,
metadata={
"help":
"if >0, then bucket source and target lengths into "
"N buckets and pad accordingly; this is useful on TPUs to minimize the number of compilations"
},
)
train_subset: str = II("dataset.train_subset")
dataset_impl: Optional[ChoiceEnum(
get_available_dataset_impl())] = II("dataset.dataset_impl")
required_seq_len_multiple: int = II("dataset.required_seq_len_multiple")
# options for reporting BLEU during validation
eval_bleu: bool = field(default=False,
metadata={"help": "evaluation with BLEU scores"})
eval_bleu_args: Optional[str] = field(
default="{}",
metadata={
"help":
'generation args for BLUE scoring, e.g., \'{"beam": 4, "lenpen": 0.6}\', as JSON string'
},
)
eval_bleu_detok: str = field(
default="space",
metadata={
"help":
"detokenize before computing BLEU (e.g., 'moses'); required if using --eval-bleu; "
"use 'space' to disable detokenization; see fairseq.data.encoders for other options"
},
)
eval_bleu_detok_args: Optional[str] = field(
default="{}",
metadata={
"help":
"args for building the tokenizer, if needed, as JSON string"
},
)
eval_tokenized_bleu: bool = field(
default=False,
metadata={"help": "compute tokenized BLEU instead of sacrebleu"})
eval_bleu_remove_bpe: Optional[str] = field(
default=None,
metadata={
"help": "remove BPE before computing BLEU",
"argparse_const": "@@ ",
},
)
eval_bleu_print_samples: bool = field(
default=False,
metadata={"help": "print sample generations during validation"})
class DatasetConfig(FairseqDataclass):
num_workers: int = field(
default=1,
metadata={"help": "how many subprocesses to use for data loading"})
skip_invalid_size_inputs_valid_test: bool = field(
default=False,
metadata={
"help": "ignore too long or too short lines in valid and test set"
},
)
max_tokens: Optional[int] = field(
default=None, metadata={"help": "maximum number of tokens in a batch"})
batch_size: Optional[int] = field(
default=None,
metadata={
"help": "number of examples in a batch",
"argparse_alias": "--max-sentences",
},
)
required_batch_size_multiple: int = field(
default=8,
metadata={"help": "batch size will be a multiplier of this value"})
required_seq_len_multiple: int = field(
default=1,
metadata={
"help":
"maximum sequence length in batch will be a multiplier of this value"
},
)
dataset_impl: Optional[DATASET_IMPL_CHOICES] = field(
default=None, metadata={"help": "output dataset implementation"})
data_buffer_size: int = field(
default=10, metadata={"help": "Number of batches to preload"})
train_subset: str = field(
default="train",
metadata={
"help": "data subset to use for training (e.g. train, valid, test)"
},
)
valid_subset: str = field(
default="valid",
metadata={
"help":
"comma separated list of data subsets to use for validation"
" (e.g. train, valid, test)"
},
)
combine_valid_subsets: Optional[bool] = field(
default=None,
metadata={
"help":
"comma separated list of data subsets to use for validation"
" (e.g. train, valid, test)",
"argparse_alias": "--combine-val",
},
)
ignore_unused_valid_subsets: Optional[bool] = field(
default=False,
metadata={"help": "do not raise error if valid subsets are ignored"},
)
validate_interval: int = field(
default=1, metadata={"help": "validate every N epochs"})
validate_interval_updates: int = field(
default=0, metadata={"help": "validate every N updates"})
validate_after_updates: int = field(
default=0,
metadata={"help": "dont validate until reaching this many updates"})
fixed_validation_seed: Optional[int] = field(
default=None,
metadata={"help": "specified random seed for validation"})
disable_validation: bool = field(default=False,
metadata={"help": "disable validation"})
max_tokens_valid: Optional[int] = field(
default=II("dataset.max_tokens"),
metadata={
"help":
"maximum number of tokens in a validation batch"
" (defaults to --max-tokens)"
},
)
batch_size_valid: Optional[int] = field(
default=II("dataset.batch_size"),
metadata={
"help":
"batch size of the validation batch (defaults to --batch-size)",
"argparse_alias": "--max-sentences-valid",
},
)
max_valid_steps: Optional[int] = field(
default=None,
metadata={
"help": "How many batches to evaluate",
"argparse_alias": "--nval"
},
)
curriculum: int = field(
default=0,
metadata={"help": "don't shuffle batches for first N epochs"})
gen_subset: str = field(
default="test",
metadata={"help": "data subset to generate (train, valid, test)"},
)
num_shards: int = field(
default=1, metadata={"help": "shard generation over N shards"})
shard_id: int = field(
default=0,
metadata={"help": "id of the shard to generate (id < num_shards)"})
grouped_shuffling: bool = field(
default=False,
metadata={
"help":
"shuffle batches in groups of num_shards to enable similar sequence lengths on each GPU worker when batches are sorted by length",
},
)
update_epoch_batch_itr: bool = field(
default=II("dataset.grouped_shuffling"),
metadata={
"help":
"if true then prevents the reuse the epoch batch iterator by setting can_reuse_epoch_itr to false, defaults to --grouped-shuffling )",
},
)
update_ordered_indices_seed: bool = field(
default=False,
metadata={
"help":
"if true then increment seed with epoch for getting batch iterators, defautls to False.",
},
)
class DistributedTrainingConfig(FairseqDataclass):
distributed_world_size: int = field(
default=max(1, torch.cuda.device_count()),
metadata={
"help":
"total number of GPUs across all nodes (default: all visible GPUs)"
},
)
distributed_num_procs: Optional[int] = field(
default=max(1, torch.cuda.device_count()),
metadata={
"help":
"total number of processes to fork (default: all visible GPUs)"
},
)
distributed_rank: Optional[int] = field(
default=0, metadata={"help": "rank of the current worker"})
distributed_backend: str = field(default="nccl",
metadata={"help": "distributed backend"})
distributed_init_method: Optional[str] = field(
default=None,
metadata={
"help":
"typically tcp://hostname:port that will be used to "
"establish initial connetion"
},
)
distributed_port: int = field(
default=-1,
metadata={
"help":
"port number (not required if using --distributed-init-method)"
},
)
device_id: int = field(
default=os.getenv("LOCAL_RANK", 0),
metadata={
"help":
"which GPU to use (by default looks for $LOCAL_RANK, usually configured automatically)",
"argparse_alias": "--local_rank",
},
)
distributed_no_spawn: bool = field(
default=False,
metadata={
"help":
"do not spawn multiple processes even if multiple GPUs are visible"
},
)
ddp_backend: DDP_BACKEND_CHOICES = field(
default="pytorch_ddp",
metadata={"help": "DistributedDataParallel backend"})
ddp_comm_hook: DDP_COMM_HOOK_CHOICES = field(
default="none", metadata={"help": "communication hook"})
bucket_cap_mb: int = field(default=25,
metadata={"help": "bucket size for reduction"})
fix_batches_to_gpus: bool = field(
default=False,
metadata={
"help":
"don't shuffle batches between GPUs; this reduces overall "
"randomness and may affect precision but avoids the cost of re-reading the data"
},
)
find_unused_parameters: bool = field(
default=False,
metadata={
"help":
"disable unused parameter detection (not applicable to "
"--ddp-backend=legacy_ddp)"
},
)
gradient_as_bucket_view: bool = field(
default=False,
metadata={
"help":
"when set to True, gradients will be views pointing to different offsets of allreduce communication buckets. This can reduce peak memory usage, where the saved memory size will be equal to the total gradients size. "
"--gradient-as-bucket-view=gradient_as_bucket_view)"
},
)
fast_stat_sync: bool = field(
default=False,
metadata={"help": "[deprecated] this is now defined per Criterion"},
)
heartbeat_timeout: int = field(
default=-1,
metadata={
"help":
"kill the job if no progress is made in N seconds; "
"set to -1 to disable"
},
)
broadcast_buffers: bool = field(
default=False,
metadata={
"help":
"Copy non-trainable parameters between GPUs, such as "
"batchnorm population statistics"
},
)
slowmo_momentum: Optional[float] = field(
default=None,
metadata={
"help":
"SlowMo momentum term; by default use 0.0 for 16 GPUs, "
"0.2 for 32 GPUs; 0.5 for 64 GPUs, 0.6 for > 64 GPUs"
},
)
slowmo_base_algorithm: str = field(
default="localsgd",
metadata={
"help":
"Base algorithm. Either 'localsgd' or 'sgp'. Please refer "
"to the documentation of 'slowmo_base_algorithm' parameter in "
"https://fairscale.readthedocs.io/en/latest/api/experimental/nn/slowmo_ddp.html "
"for more details"
},
)
localsgd_frequency: int = field(
default=3, metadata={"help": "Local SGD allreduce frequency"})
nprocs_per_node: int = field(
default=max(1, torch.cuda.device_count()),
metadata={
"help":
"number of GPUs in each node. An allreduce operation across GPUs in "
"a node is very fast. Hence, we do allreduce across GPUs in a node, "
"and gossip across different nodes"
},
)
pipeline_model_parallel: bool = field(
default=False,
metadata={
"help": "if set, use pipeline model parallelism across GPUs"
},
)
pipeline_balance: Optional[str] = field(
default=None,
metadata={
"help":
"partition the model into N_K pieces, where each piece "
"contains N_i layers. The sum(args.pipeline_balance) "
"should equal the total number of layers in the model"
},
)
pipeline_devices: Optional[str] = field(
default=None,
metadata={
"help":
"a list of device indices indicating which device to place "
"each of the N_K partitions. The length of this list should "
"equal the length of the --pipeline-balance argument"
},
)
pipeline_chunks: Optional[int] = field(
default=0,
metadata={"help": "microbatch count for pipeline model parallelism"})
pipeline_encoder_balance: Optional[str] = field(
default=None,
metadata={
"help":
"partition the pipeline parallel encoder into N_K pieces, where each piece "
"contains N_i layers. The sum(args.pipeline_encoder_balance) "
"should equal the total number of encoder layers in the model"
},
)
pipeline_encoder_devices: Optional[str] = field(
default=None,
metadata={
"help":
"a list of device indices indicating which device to place "
"each of the N_K partitions. The length of this list should "
"equal the length of the --pipeline-encoder-balance argument"
},
)
pipeline_decoder_balance: Optional[str] = field(
default=None,
metadata={
"help":
"partition the pipeline parallel decoder into N_K pieces, where each piece "
"contains N_i layers. The sum(args.pipeline_decoder_balance) "
"should equal the total number of decoder layers in the model"
},
)
pipeline_decoder_devices: Optional[str] = field(
default=None,
metadata={
"help":
"a list of device indices indicating which device to place "
"each of the N_K partitions. The length of this list should "
"equal the length of the --pipeline-decoder-balance argument"
},
)
pipeline_checkpoint: PIPELINE_CHECKPOINT_CHOICES = field(
default="never",
metadata={"help": "checkpointing mode for pipeline model parallelism"},
)
zero_sharding: ZERO_SHARDING_CHOICES = field(
default="none", metadata={"help": "ZeRO sharding"})
fp16: bool = II("common.fp16")
memory_efficient_fp16: bool = II("common.memory_efficient_fp16")
tpu: bool = II("common.tpu")
# configuration for --ddp-backend=fully_sharded
no_reshard_after_forward: bool = field(
default=False,
metadata={"help": "don't reshard parameters after forward pass"},
)
fp32_reduce_scatter: bool = field(
default=False,
metadata={"help": "reduce-scatter grads in FP32"},
)
cpu_offload: bool = field(default=False,
metadata={"help": "offload FP32 params to CPU"})
use_sharded_state: bool = field(
default=False,
metadata={"help": "use sharded checkpoint files"},
)
not_fsdp_flatten_parameters: bool = field(
default=False,
metadata={"help": "not flatten parameter param for fsdp"},
)
class StructuredInterpolationValidationError:
x: Optional[int] = None
y: int = II(".x")
class DistributedTrainingConfig(FairseqDataclass):
distributed_world_size: int = field(
default=max(1, torch.cuda.device_count()),
metadata={
"help": "total number of GPUs across all nodes (default: all visible GPUs)"
},
)
distributed_rank: Optional[int] = field(
default=0, metadata={"help": "rank of the current worker"}
)
distributed_backend: str = field(
default="nccl", metadata={"help": "distributed backend"}
)
distributed_init_method: Optional[str] = field(
default=None,
metadata={
"help": "typically tcp://hostname:port that will be used to "
"establish initial connetion"
},
)
distributed_port: int = field(
default=-1,
metadata={
"help": "port number (not required if using --distributed-init-method)"
},
)
device_id: int = field(
default=0,
metadata={"help": "which GPU to use (usually configured automatically)"},
)
local_rank: int = field(
default=0,
metadata={"help": "which GPU to use (usually configured automatically)"},
)
distributed_no_spawn: bool = field(
default=False,
metadata={
"help": "do not spawn multiple processes even if multiple GPUs are visible"
},
)
ddp_backend: DDP_BACKEND_CHOICES = field(
default="c10d", metadata={"help": "DistributedDataParallel backend"}
)
bucket_cap_mb: int = field(
default=25, metadata={"help": "bucket size for reduction"}
)
fix_batches_to_gpus: bool = field(
default=False,
metadata={
"help": "don't shuffle batches between GPUs; this reduces overall "
"randomness and may affect precision but avoids the cost of re-reading the data"
},
)
find_unused_parameters: bool = field(
default=False,
metadata={
"help": "disable unused parameter detection (not applicable to "
"no_c10d ddp-backend"
},
)
fast_stat_sync: bool = field(
default=False,
metadata={"help": "[deprecated] this is now defined per Criterion"},
)
broadcast_buffers: bool = field(
default=False,
metadata={
"help": "Copy non-trainable parameters between GPUs, such as "
"batchnorm population statistics"
},
)
distributed_wrapper: DISTRIBUTED_WRAPPER_CHOICES = field(
default="DDP", metadata={"help": "DistributedDataParallel backend"}
)
slowmo_momentum: Optional[float] = field(
default=None,
metadata={
"help": "SlowMo momentum term; by default use 0.0 for 16 GPUs, "
"0.2 for 32 GPUs; 0.5 for 64 GPUs, 0.6 for > 64 GPUs"
},
)
slowmo_algorithm: str = field(
default="LocalSGD", metadata={"help": "whether to use LocalSGD or SGP"}
)
localsgd_frequency: int = field(
default=3, metadata={"help": "Local SGD allreduce frequency"}
)
nprocs_per_node: int = field(
default=max(1, torch.cuda.device_count()),
metadata={
"help": "number of GPUs in each node. An allreduce operation across GPUs in "
"a node is very fast. Hence, we do allreduce across GPUs in a node, "
"and gossip across different nodes"
},
)
pipeline_model_parallel: bool = field(
default=False,
metadata={"help": "if set, use pipeline model parallelism across GPUs"},
)
pipeline_balance: Optional[str] = field(
default=None,
metadata={
"help": "partition the model into N_K pieces, where each piece "
"contains N_i layers. The sum(args.pipeline_balance) "
"should equal the total number of layers in the model"
},
)
pipeline_devices: Optional[str] = field(
default=None,
metadata={
"help": "a list of device indices indicating which device to place "
"each of the N_K partitions. The length of this list should "
"equal the length of the --pipeline-balance argument"
},
)
pipeline_chunks: Optional[int] = field(
default=0, metadata={"help": "microbatch count for pipeline model parallelism"}
)
pipeline_encoder_balance: Optional[str] = field(
default=None,
metadata={
"help": "partition the pipeline parallel encoder into N_K pieces, where each piece "
"contains N_i layers. The sum(args.pipeline_encoder_balance) "
"should equal the total number of encoder layers in the model"
},
)
pipeline_encoder_devices: Optional[str] = field(
default=None,
metadata={
"help": "a list of device indices indicating which device to place "
"each of the N_K partitions. The length of this list should "
"equal the length of the --pipeline-encoder-balance argument"
},
)
pipeline_decoder_balance: Optional[str] = field(
default=None,
metadata={
"help": "partition the pipeline parallel decoder into N_K pieces, where each piece "
"contains N_i layers. The sum(args.pipeline_decoder_balance) "
"should equal the total number of decoder layers in the model"
},
)
pipeline_decoder_devices: Optional[str] = field(
default=None,
metadata={
"help": "a list of device indices indicating which device to place "
"each of the N_K partitions. The length of this list should "
"equal the length of the --pipeline-decoder-balance argument"
},
)
pipeline_checkpoint: PIPELINE_CHECKPOINT_CHOICES = field(
default="never",
metadata={"help": "checkpointing mode for pipeline model parallelism"},
)
zero_sharding: ZERO_SHARDING_CHOICES = field(
default="none", metadata={"help": "ZeRO sharding"}
)
tpu: bool = II("common.tpu")
class CheckpointConfig(FairseqDataclass):
save_dir: str = field(default="checkpoints",
metadata={"help": "path to save checkpoints"})
restore_file: str = field(
default="checkpoint_last.pt",
metadata={
"help":
"filename from which to load checkpoint "
"(default: <save-dir>/checkpoint_last.pt"
},
)
finetune_from_model: Optional[str] = field(
default=None,
metadata={
"help":
"finetune from a pretrained model; note that meters and lr scheduler will be reset"
},
)
reset_dataloader: bool = field(
default=False,
metadata={
"help":
"if set, does not reload dataloader state from the checkpoint"
},
)
reset_lr_scheduler: bool = field(
default=False,
metadata={
"help":
"if set, does not load lr scheduler state from the checkpoint"
},
)
reset_meters: bool = field(
default=False,
metadata={"help": "if set, does not load meters from the checkpoint"},
)
reset_optimizer: bool = field(
default=False,
metadata={
"help": "if set, does not load optimizer state from the checkpoint"
},
)
optimizer_overrides: str = field(
default="{}",
metadata={
"help":
"a dictionary used to override optimizer args when loading a checkpoint"
},
)
save_interval: int = field(
default=1, metadata={"help": "save a checkpoint every N epochs"})
save_interval_updates: int = field(
default=0,
metadata={"help": "save a checkpoint (and validate) every N updates"})
keep_interval_updates: int = field(
default=-1,
metadata={
"help":
"keep the last N checkpoints saved with --save-interval-updates"
},
)
keep_interval_updates_pattern: int = field(
default=-1,
metadata={
"help":
"when used with --keep-interval-updates, skips deleting "
"any checkpoints with update X where "
"X % keep_interval_updates_pattern == 0"
},
)
keep_last_epochs: int = field(
default=-1, metadata={"help": "keep last N epoch checkpoints"})
keep_best_checkpoints: int = field(
default=-1,
metadata={"help": "keep best N checkpoints based on scores"})
no_save: bool = field(
default=False, metadata={"help": "don't save models or checkpoints"})
no_epoch_checkpoints: bool = field(
default=False,
metadata={"help": "only store last and best checkpoints"})
no_last_checkpoints: bool = field(
default=False, metadata={"help": "don't store last checkpoints"})
no_save_optimizer_state: bool = field(
default=False,
metadata={"help": "don't save optimizer-state as part of checkpoint"},
)
best_checkpoint_metric: str = field(
default="loss",
metadata={"help": 'metric to use for saving "best" checkpoints'})
maximize_best_checkpoint_metric: bool = field(
default=False,
metadata={
"help":
'select the largest metric value for saving "best" checkpoints'
},
)
patience: int = field(
default=-1,
metadata={
"help": ("early stop training if valid performance doesn't "
"improve for N consecutive validation runs; note "
"that this is influenced by --validate-interval")
},
)
checkpoint_suffix: str = field(
default="",
metadata={"help": "suffix to add to the checkpoint file name"})
checkpoint_shard_count: int = field(
default=1,
metadata={
"help":
"Number of shards containing the checkpoint - "
"if the checkpoint is over 300GB, it is preferable "
"to split it into shards to prevent OOM on CPU while loading "
"the checkpoint"
},
)
load_checkpoint_on_all_dp_ranks: bool = field(
default=False,
metadata={
"help":
"load checkpoints on all data parallel devices "
"(default: only load on rank 0 and broadcast to other devices)"
},
)
write_checkpoints_asynchronously: bool = field(
default=False,
metadata={
"help": ("Write checkpoints asynchronously in a separate "
"thread. NOTE: This feature is currently being tested."),
"argparse_alias":
"--save-async",
},
)
model_parallel_size: int = II("common.model_parallel_size")
class CheckpointConfig(FairseqDataclass):
save_dir: str = field(
default="checkpoints", metadata={"help": "path to save checkpoints"}
)
restore_file: str = field(
default="checkpoint_last.pt",
metadata={
"help": "filename from which to load checkpoint "
"(default: <save-dir>/checkpoint_last.pt"
},
)
finetune_from_model: Optional[str] = field(
default=None,
metadata={
"help": "finetune from a pretrained model; note that meters and lr scheduler will be reset"
},
)
reset_dataloader: bool = field(
default=False,
metadata={
"help": "if set, does not reload dataloader state from the checkpoint"
},
)
reset_lr_scheduler: bool = field(
default=False,
metadata={
"help": "if set, does not load lr scheduler state from the checkpoint"
},
)
reset_meters: bool = field(
default=False,
metadata={"help": "if set, does not load meters from the checkpoint"},
)
reset_optimizer: bool = field(
default=False,
metadata={"help": "if set, does not load optimizer state from the checkpoint"},
)
optimizer_overrides: str = field(
default="{}",
metadata={
"help": "a dictionary used to override optimizer args when loading a checkpoint"
},
)
save_interval: int = field(
default=1, metadata={"help": "save a checkpoint every N epochs"}
)
save_interval_updates: int = field(
default=0, metadata={"help": "save a checkpoint (and validate) every N updates"}
)
keep_interval_updates: int = field(
default=-1,
metadata={
"help": "keep the last N checkpoints saved with --save-interval-updates"
},
)
keep_last_epochs: int = field(
default=-1, metadata={"help": "keep last N epoch checkpoints"}
)
keep_best_checkpoints: int = field(
default=-1, metadata={"help": "keep best N checkpoints based on scores"}
)
no_save: bool = field(
default=False, metadata={"help": "don't save models or checkpoints"}
)
no_epoch_checkpoints: bool = field(
default=False, metadata={"help": "only store last and best checkpoints"}
)
no_last_checkpoints: bool = field(
default=False, metadata={"help": "don't store last checkpoints"}
)
no_save_optimizer_state: bool = field(
default=False,
metadata={"help": "don't save optimizer-state as part of checkpoint"},
)
best_checkpoint_metric: str = field(
default="loss", metadata={"help": 'metric to use for saving "best" checkpoints'}
)
maximize_best_checkpoint_metric: bool = field(
default=False,
metadata={
"help": 'select the largest metric value for saving "best" checkpoints'
},
)
patience: int = field(
default=-1,
metadata={
"help": (
"early stop training if valid performance doesn't "
"improve for N consecutive validation runs; note "
"that this is influenced by --validate-interval"
)
},
)
checkpoint_suffix: str = field(
default="", metadata={"help": "suffix to add to the checkpoint file name"}
)
checkpoint_shard_count: int = field(
default=1,
metadata={
"help": "Number of shards containing the checkpoint - "
"if the checkpoint is over 300GB, it is preferable "
"to split it into shards to prevent OOM on CPU while loading "
"the checkpoint"
},
)
model_parallel_size: int = II("common.model_parallel_size")
distributed_rank: int = II("distributed_training.distributed_rank")
def __post_init__(self):
# II doesn't work if we are just creating the object outside of hydra so fix that
if self.input_dim == II("model.decoder.embed_dim"):
self.input_dim = self.embed_dim
if self.output_dim == II("model.decoder.embed_dim"):
self.output_dim = self.embed_dim
class InterpolationDict:
dict: Dict[str, int] = II("optimization.lr")
class Wav2BertConfig(FairseqDataclass):
w2v_path: str = field(default=MISSING,
metadata={"help": "path to wav2vec 2.0 model"})
no_pretrained_weights: bool = field(
default=False,
metadata={"help": "if true, does not load pretrained weights"})
dropout_input: float = field(
default=0.0,
metadata={"help": "dropout to apply to the input (after feat extr)"},
)
final_dropout: float = field(
default=0.0,
metadata={
"help": "dropout after transformer and before final projection"
},
)
dropout: float = field(
default=0.0,
metadata={"help": "dropout probability inside wav2vec 2.0 model"})
attention_dropout: float = field(
default=0.0,
metadata={
"help":
"dropout probability for attention weights inside wav2vec 2.0 model"
},
)
activation_dropout: float = field(
default=0.0,
metadata={
"help":
"dropout probability after activation in FFN inside wav2vec 2.0 model"
},
)
# masking
apply_mask: bool = field(
default=False, metadata={"help": "apply masking during fine-tuning"})
mask_length: int = field(
default=10,
metadata={"help": "repeat the mask indices multiple times"})
mask_prob: float = field(
default=0.5,
metadata={
"help":
"probability of replacing a token with mask (normalized by length)"
},
)
mask_selection: MASKING_DISTRIBUTION_CHOICES = field(
default="static", metadata={"help": "how to choose masks"})
mask_other: float = field(
default=0,
metadata={
"help":
"secondary mask argument (used for more complex distributions), "
"see help in compute_mask_indices"
},
)
no_mask_overlap: bool = field(
default=False, metadata={"help": "whether to allow masks to overlap"})
# channel masking
mask_channel_length: int = field(
default=10,
metadata={"help": "length of the mask for features (channels)"})
mask_channel_prob: float = field(
default=0.0,
metadata={"help": "probability of replacing a feature with 0"})
mask_channel_selection: MASKING_DISTRIBUTION_CHOICES = field(
default="static",
metadata={"help": "how to choose mask length for channel masking"},
)
mask_channel_other: float = field(
default=0,
metadata={
"help":
"secondary mask argument (used for more complex distributions), "
"see help in compute_mask_indicesh"
},
)
no_mask_channel_overlap: bool = field(
default=False,
metadata={"help": "whether to allow channel masks to overlap"})
freeze_finetune_updates: int = field(
default=0,
metadata={"help": "dont finetune wav2vec for this many updates"})
feature_grad_mult: float = field(
default=0.0,
metadata={"help": "reset feature grad mult in wav2vec 2.0 to this"})
layerdrop: float = field(
default=0.0,
metadata={"help": "probability of dropping a layer in wav2vec 2.0"})
mask_min_space: Optional[int] = field(
default=1,
metadata={
"help": "min space between spans (if no overlap is enabled)"
},
)
mask_channel_min_space: Optional[int] = field(
default=1,
metadata={
"help": "min space between spans (if no overlap is enabled)"
},
)
conv_feature_layers: Optional[str] = field(
default=
"[(512, 10, 5)] + [(512, 3, 2)] * 4 + [(512,2,2)] + [(512,2,2)]",
metadata={
"help": ("string describing convolutional feature extraction "
"layers in form of a python list that contains "
"[(dim, kernel_size, stride), ...]"),
},
)
encoder_embed_dim: Optional[int] = field(
default=768, metadata={"help": "encoder embedding dimension"})
normalize: bool = II("task.normalize")
data: str = II("task.data")
# this holds the loaded wav2vec args
w2v_args: Any = None
fix_extractor: bool = False
bert_path: str = field(
default="",
metadata={"help": "path of bart model"},
)
fix_encoder: bool = False
fix_decoder: bool = False
autoregressive: bool = II("task.autoregressive")
wav2vec_weight: float = 0.
wav2bert_weight: float = 1.
class InferredW2vConfig:
# The following are needed to precompute mask and mask channel indices
# before model's forward.
mask_length: Optional[int] = II("model.mask_length")
mask_prob: Optional[float] = II("model.mask_prob")
mask_selection: Optional[str] = II("model.mask_selection")
mask_other: Optional[float] = II("model.mask_other")
no_mask_overlap: Optional[bool] = II("model.no_mask_overlap")
mask_min_space: Optional[int] = II("model.mask_min_space")
mask_channel_length: Optional[int] = II("model.mask_channel_length")
mask_channel_prob: Optional[float] = II("model.mask_channel_prob")
mask_channel_selection: Optional[str] = II("model.mask_channel_selection")
mask_channel_other: Optional[float] = II("model.mask_channel_other")
no_mask_channel_overlap: Optional[bool] = II("model.no_mask_channel_overlap")
mask_channel_min_space: Optional[int] = II("model.mask_channel_min_space")
conv_feature_layers: Optional[str] = II("model.conv_feature_layers")
encoder_embed_dim: Optional[int] = II("model.encoder_embed_dim")
class RelativeInterpolation:
x: int = 100
y: int = 200
z1: int = II(".x")
z2: str = SI("${.x}_${.y}")
class Wav2Vec2AsrConfig(FairseqDataclass):
w2v_path: str = field(default=MISSING,
metadata={"help": "path to wav2vec 2.0 model"})
no_pretrained_weights: bool = field(
default=False,
metadata={"help": "if true, does not load pretrained weights"})
dropout_input: float = field(
default=0.0,
metadata={"help": "dropout to apply to the input (after feat extr)"},
)
final_dropout: float = field(
default=0.0,
metadata={
"help": "dropout after transformer and before final projection"
},
)
dropout: float = field(
default=0.0,
metadata={"help": "dropout probability inside wav2vec 2.0 model"})
attention_dropout: float = field(
default=0.0,
metadata={
"help":
"dropout probability for attention weights inside wav2vec 2.0 model"
},
)
activation_dropout: float = field(
default=0.0,
metadata={
"help":
"dropout probability after activation in FFN inside wav2vec 2.0 model"
},
)
# masking
apply_mask: bool = field(
default=False, metadata={"help": "apply masking during fine-tuning"})
mask_length: int = field(
default=10,
metadata={"help": "repeat the mask indices multiple times"})
mask_prob: float = field(
default=0.5,
metadata={
"help":
"probability of replacing a token with mask (normalized by length)"
},
)
mask_selection: MASKING_DISTRIBUTION_CHOICES = field(
default="static", metadata={"help": "how to choose masks"})
mask_other: float = field(
default=0,
metadata={
"help":
"secondary mask argument (used for more complex distributions), "
"see help in compute_mask_indices"
},
)
no_mask_overlap: bool = field(
default=False, metadata={"help": "whether to allow masks to overlap"})
# channel masking
mask_channel_length: int = field(
default=10,
metadata={"help": "length of the mask for features (channels)"})
mask_channel_prob: float = field(
default=0.0,
metadata={"help": "probability of replacing a feature with 0"})
mask_channel_selection: MASKING_DISTRIBUTION_CHOICES = field(
default="static",
metadata={"help": "how to choose mask length for channel masking"},
)
mask_channel_other: float = field(
default=0,
metadata={
"help":
"secondary mask argument (used for more complex distributions), "
"see help in compute_mask_indicesh"
},
)
no_mask_channel_overlap: bool = field(
default=False,
metadata={"help": "whether to allow channel masks to overlap"})
freeze_finetune_updates: int = field(
default=0,
metadata={"help": "dont finetune wav2vec for this many updates"})
feature_grad_mult: float = field(
default=0.0,
metadata={"help": "reset feature grad mult in wav2vec 2.0 to this"})
layerdrop: float = field(
default=0.0,
metadata={"help": "probability of dropping a layer in wav2vec 2.0"})
normalize: bool = II("task.normalize")
data: str = II("task.data")
# this holds the loaded wav2vec args
w2v_args: Any = None
class HubertConfig(FairseqDataclass):
label_rate: int = II("task.label_rate")
extractor_mode: EXTRACTOR_MODE_CHOICES = field(
default="default",
metadata={
"help":
"mode for feature extractor. default has a single group "
"norm with d groups in the first conv block, whereas layer_norm "
"has layer norms in every block (meant to use with normalize=True)"
},
)
encoder_layers: int = field(
default=12, metadata={"help": "num encoder layers in the transformer"})
encoder_embed_dim: int = field(
default=768, metadata={"help": "encoder embedding dimension"})
encoder_ffn_embed_dim: int = field(
default=3072, metadata={"help": "encoder embedding dimension for FFN"})
encoder_attention_heads: int = field(
default=12, metadata={"help": "num encoder attention heads"})
activation_fn: ChoiceEnum(utils.get_available_activation_fns()) = field(
default="gelu", metadata={"help": "activation function to use"})
# dropouts
dropout: float = field(
default=0.1,
metadata={"help": "dropout probability for the transformer"},
)
attention_dropout: float = field(
default=0.1,
metadata={"help": "dropout probability for attention weights"},
)
activation_dropout: float = field(
default=0.0,
metadata={"help": "dropout probability after activation in FFN"},
)
encoder_layerdrop: float = field(
default=0.0,
metadata={"help": "probability of dropping a tarnsformer layer"},
)
dropout_input: float = field(
default=0.0,
metadata={"help": "dropout to apply to the input (after feat extr)"},
)
dropout_features: float = field(
default=0.0,
metadata={
"help": "dropout to apply to the features (after feat extr)"
},
)
final_dim: int = field(
default=0,
metadata={
"help":
"project final representations and targets to this many "
"dimensions. set to encoder_embed_dim is <= 0"
},
)
untie_final_proj: bool = field(
default=False,
metadata={"help": "use separate projection for each target"},
)
layer_norm_first: bool = field(
default=False,
metadata={"help": "apply layernorm first in the transformer"},
)
conv_feature_layers: str = field(
default="[(512,10,5)] + [(512,3,2)] * 4 + [(512,2,2)] * 2",
metadata={
"help":
"string describing convolutional feature extraction "
"layers in form of a python list that contains "
"[(dim, kernel_size, stride), ...]"
},
)
conv_bias: bool = field(default=False,
metadata={"help": "include bias in conv encoder"})
logit_temp: float = field(
default=0.1, metadata={"help": "temperature to divide logits by"})
target_glu: bool = field(
default=False, metadata={"help": "adds projection + glu to targets"})
feature_grad_mult: float = field(
default=1.0,
metadata={"help": "multiply feature extractor var grads by this"},
)
# masking
mask_length: int = field(default=10, metadata={"help": "mask length"})
mask_prob: float = field(
default=0.65,
metadata={"help": "probability of replacing a token with mask"},
)
mask_selection: MASKING_DISTRIBUTION_CHOICES = field(
default="static", metadata={"help": "how to choose mask length"})
mask_other: float = field(
default=0,
metadata={
"help":
"secondary mask argument "
"(used for more complex distributions), "
"see help in compute_mask_indicesh"
},
)
no_mask_overlap: bool = field(
default=False, metadata={"help": "whether to allow masks to overlap"})
mask_min_space: int = field(
default=1,
metadata={
"help": "min space between spans (if no overlap is enabled)"
},
)
# channel masking
mask_channel_length: int = field(
default=10,
metadata={"help": "length of the mask for features (channels)"},
)
mask_channel_prob: float = field(
default=0.0,
metadata={"help": "probability of replacing a feature with 0"},
)
mask_channel_selection: MASKING_DISTRIBUTION_CHOICES = field(
default="static",
metadata={"help": "how to choose mask length for channel masking"},
)
mask_channel_other: float = field(
default=0,
metadata={
"help":
"secondary mask argument "
"(used for more complex distributions), "
"see help in compute_mask_indicesh"
},
)
no_mask_channel_overlap: bool = field(
default=False,
metadata={"help": "whether to allow channel masks to overlap"},
)
mask_channel_min_space: int = field(
default=1,
metadata={
"help": "min space between spans (if no overlap is enabled)"
},
)
# positional embeddings
conv_pos: int = field(
default=128,
metadata={
"help": "number of filters for convolutional positional embeddings"
},
)
conv_pos_groups: int = field(
default=16,
metadata={
"help": "number of groups for convolutional positional embedding"
},
)
latent_temp: Tuple[float, float, float] = field(
default=(2, 0.5, 0.999995),
metadata={"help": "legacy (to be removed)"},
)
# loss computation
skip_masked: bool = field(
default=False,
metadata={"help": "skip computing losses over masked frames"},
)
skip_nomask: bool = field(
default=False,
metadata={"help": "skip computing losses over unmasked frames"},
)
class TransformerLanguageModelConfig(FairseqDataclass):
activation_fn: ChoiceEnum(utils.get_available_activation_fns()) = field(
default="relu", metadata={"help": "activation function to use"})
dropout: float = field(default=0.1,
metadata={"help": "dropout probability"})
attention_dropout: float = field(
default=0.0,
metadata={"help": "dropout probability for attention weights"})
activation_dropout: float = field(
default=0.0,
metadata={"help": "dropout probability after activation in FFN."})
relu_dropout: float = field(
default=0.0,
metadata={"help": "dropout probability after activation in FFN."})
decoder_embed_dim: int = field(
default=512, metadata={"help": "decoder embedding dimension"})
decoder_output_dim: int = field(
default=512, metadata={"help": "decoder output dimension"})
decoder_input_dim: int = field(
default=512, metadata={"help": "decoder input dimension"})
decoder_ffn_embed_dim: int = field(
default=2048, metadata={"help": "decoder embedding dimension for FFN"})
decoder_layers: int = field(default=6,
metadata={"help": "num decoder layers"})
decoder_attention_heads: int = field(
default=8, metadata={"help": "num decoder attention heads"})
decoder_normalize_before: bool = field(
default=False,
metadata={"help": "apply layernorm before each decoder block"})
no_decoder_final_norm: bool = field(
default=False,
metadata={
"help": "don't add an extra layernorm after the last decoder block"
},
)
adaptive_softmax_cutoff: Optional[str] = field(
default=None,
metadata={
"help":
"comma separated list of adaptive softmax cutoff points. "
"Must be used with adaptive_loss criterion"
},
)
adaptive_softmax_dropout: float = field(
default=0,
metadata={
"help": "sets adaptive softmax dropout for the tail projections"
},
)
adaptive_softmax_factor: float = field(
default=4, metadata={"help": "adaptive input factor"})
no_token_positional_embeddings: bool = field(
default=False,
metadata={
"help":
"if set, disables positional embeddings (outside self attention)"
},
)
share_decoder_input_output_embed: bool = field(
default=False,
metadata={"help": "share decoder input and output embeddings"})
character_embeddings: bool = field(
default=False,
metadata={
"help":
"if set, uses character embedding convolutions to produce token embeddings"
},
)
character_filters: str = field(
default=
"[(1, 64), (2, 128), (3, 192), (4, 256), (5, 256), (6, 256), (7, 256)]",
metadata={"help": "size of character embeddings"},
)
character_embedding_dim: int = field(
default=4, metadata={"help": "size of character embeddings"})
char_embedder_highway_layers: int = field(
default=2,
metadata={
"help": "number of highway layers for character token embeddder"
},
)
adaptive_input: bool = field(
default=False, metadata={"help": "if set, uses adaptive input"})
adaptive_input_factor: float = field(
default=4, metadata={"help": "adaptive input factor"})
adaptive_input_cutoff: Optional[str] = field(
default=None,
metadata={
"help": "comma separated list of adaptive input cutoff points."
},
)
tie_adaptive_weights: bool = field(
default=False,
metadata={
"help":
"if set, ties the weights of adaptive softmax and adaptive input"
},
)
tie_adaptive_proj: bool = field(
default=False,
metadata={
"help":
"if set, ties the projection weights of adaptive softmax and adaptive input"
},
)
decoder_learned_pos: bool = field(
default=False,
metadata={"help": "use learned positional embeddings in the decoder"},
)
decoder_layerdrop: float = field(
default=0.0, metadata={"help": "LayerDrop probability for decoder"})
decoder_layers_to_keep: Optional[str] = field(
default=None,
metadata={
"help":
"which layers to *keep* when pruning as a comma-separated list"
},
)
layernorm_embedding: bool = field(
default=False, metadata={"help": "add layernorm to embedding"})
no_scale_embedding: bool = field(
default=False, metadata={"help": "if True, dont scale embeddings"})
quant_noise_pq: float = field(
default=0.0,
metadata={"help": "iterative PQ quantization noise at training time"},
)
quant_noise_pq_block_size: int = field(
default=8,
metadata={"help": "block size of quantization noise at training time"},
)
# TODO common var add to parent
quant_noise_scalar: float = field(
default=0.0,
metadata={
"help":
"scalar quantization noise and scalar quantization at training time"
},
)
add_bos_token: bool = II("task.add_bos_token")
tokens_per_sample: int = II("task.tokens_per_sample")
max_target_positions: Optional[int] = II("task.max_target_positions")
# TODO common var add to parent
tpu: bool = II("params.common.tpu")
class CrossEntropyCriterionConfig(FairseqDataclass):
sentence_avg: bool = II("params.optimization.sentence_avg")
