# Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # # This source code is licensed under the license found in the LICENSE file in # the root directory of this source tree. An additional grant of patent rights # can be found in the PATENTS file in the same directory. import importlib import os from fairseq import registry build_tokenizer, register_tokenizer, TOKENIZER_REGISTRY = registry.setup_registry( '--tokenizer', default=None, ) build_bpe, register_bpe, BPE_REGISTRY = registry.setup_registry( '--bpe', default=None, ) # automatically import any Python files in the transforms/ directory for file in os.listdir(os.path.dirname(__file__)): if file.endswith('.py') and not file.startswith('_'): module = file[:file.find('.py')] importlib.import_module('fairseq.data.transforms.' + module)
import importlib import os from fairseq import registry from fairseq.optim.fairseq_optimizer import FairseqOptimizer from fairseq.optim.fp16_optimizer import FP16Optimizer, MemoryEfficientFP16Optimizer __all__ = [ 'FairseqOptimizer', 'FP16Optimizer', 'MemoryEfficientFP16Optimizer', ] _build_optimizer, register_optimizer, OPTIMIZER_REGISTRY = registry.setup_registry( '--optimizer', base_class=FairseqOptimizer, default='nag', ) def build_optimizer(args, params, *extra_args, **extra_kwargs): params = list(filter(lambda p: p.requires_grad, params)) return _build_optimizer(args, params, *extra_args, **extra_kwargs) # automatically import any Python files in the optim/ directory for file in os.listdir(os.path.dirname(__file__)): if file.endswith('.py') and not file.startswith('_'): module = file[:file.find('.py')] importlib.import_module('fairseq.optim.' + module)
"""isort:skip_file""" import importlib import os from fairseq import registry from fairseq.optim.lr_scheduler.fairseq_lr_scheduler import ( # noqa FairseqLRScheduler, LegacyFairseqLRScheduler, ) from omegaconf import DictConfig ( build_lr_scheduler_, register_lr_scheduler, LR_SCHEDULER_REGISTRY, LR_SCHEDULER_DATACLASS_REGISTRY, ) = registry.setup_registry("--lr-scheduler", base_class=FairseqLRScheduler, default="fixed") def build_lr_scheduler(cfg: DictConfig, optimizer): return build_lr_scheduler_(cfg, optimizer) # automatically import any Python files in the optim/lr_scheduler/ directory for file in os.listdir(os.path.dirname(__file__)): if file.endswith(".py") and not file.startswith("_"): file_name = file[:file.find(".py")] importlib.import_module("fairseq.optim.lr_scheduler." + file_name)
# Copyright (c) Facebook, Inc. and its affiliates. # # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. import importlib import os from fairseq import registry from fairseq.criterions.fairseq_criterion import FairseqCriterion build_criterion, register_criterion, CRITERION_REGISTRY = registry.setup_registry( '--criterion', base_class=FairseqCriterion, default='cross_entropy', ) # automatically import any Python files in the criterions/ directory for file in os.listdir(os.path.dirname(__file__)): if file.endswith('.py') and not file.startswith('_'): module = file[:file.find('.py')] importlib.import_module('fairseq.criterions.' + module)
# Copyright (c) Facebook, Inc. and its affiliates. # # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. import importlib import os from fairseq import registry build_agent, register_agent, MONOTONIC_AGENT, _ = registry.setup_registry( "--agent-type") DEFAULT_EOS = "</s>" GET = 0 SEND = 1 for file in os.listdir(os.path.dirname(__file__)): if file.endswith(".py") and not file.startswith("_"): module = file[:file.find(".py")] importlib.import_module("agents." + module)
import importlib import os from fairseq import registry # ConvNets: (build_convnet, register_convnet, CONVNET_REGISTRY) = registry.setup_registry('--convnet') # Aggregators: (build_aggregator, register_aggregator, AGGREGATOR_REGISTRY) = registry.setup_registry('--aggregator') from .masked_convolution import MaskedConvolution from .pa_controller import PAController # automatically import any Python files in the criterions/ directory for file in os.listdir(os.path.dirname(__file__)): if file.endswith('.py') and not file.startswith('_'): module = file[:file.find('.py')] importlib.import_module('examples.pervasive.modules.' + module)
# Copyright (c) Facebook, Inc. and its affiliates. # # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. import importlib import os from fairseq import registry from fairseq.optim.lr_scheduler.fairseq_lr_scheduler import FairseqLRScheduler, LegacyFairseqLRScheduler # noqa LR_SCHEDULER_DATACLASS_REGISTRY = {} build_lr_scheduler, register_lr_scheduler, LR_SCHEDULER_REGISTRY = registry.setup_registry( '--lr-scheduler', base_class=FairseqLRScheduler, default='fixed', ) # automatically import any Python files in the optim/lr_scheduler/ directory for file in os.listdir(os.path.dirname(__file__)): if file.endswith('.py') and not file.startswith('_'): module = file[:file.find('.py')] importlib.import_module('fairseq.optim.lr_scheduler.' + module)
# Copyright (c) Facebook, Inc. and its affiliates. # # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. import importlib import os from fairseq import registry (build_scorer, register_scorer, SCORER_REGISTRIES, _) = registry.setup_registry('--scorer-type') for file in os.listdir(os.path.dirname(__file__)): if file.endswith('.py') and not file.startswith('_'): module = file[:file.find('.py')] importlib.import_module('scorers.' + module)
# Copyright (c) Facebook, Inc. and its affiliates. # # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. import os import importlib from fairseq import registry ( build_monotonic_attention, register_monotonic_attention, MONOTONIC_ATTENTION_REGISTRY, _, ) = registry.setup_registry("--simul-type") for file in sorted(os.listdir(os.path.dirname(__file__))): if file.endswith(".py") and not file.startswith("_"): model_name = file[:file.find(".py")] importlib.import_module("examples.simultaneous_translation.modules." + model_name)
from fairseq import registry from fairseq.optim.fairseq_optimizer import FairseqOptimizer, LegacyFairseqOptimizer # noqa from fairseq.optim.fp16_optimizer import FP16Optimizer, MemoryEfficientFP16Optimizer from fairseq.optim.bmuf import FairseqBMUF # noqa from fairseq.optim.shard import shard_ __all__ = [ 'FairseqOptimizer', 'FP16Optimizer', 'MemoryEfficientFP16Optimizer', 'shard_', ] _build_optimizer, register_optimizer, OPTIMIZER_REGISTRY = registry.setup_registry( '--optimizer', base_class=FairseqOptimizer, required=True, ) def build_optimizer(args, params, *extra_args, **extra_kwargs): if all(isinstance(p, dict) for p in params): params = [t for p in params for t in p.values()] params = list(filter(lambda p: p.requires_grad, params)) return _build_optimizer(args, params, *extra_args, **extra_kwargs) # automatically import any Python files in the optim/ directory for file in os.listdir(os.path.dirname(__file__)): if file.endswith('.py') and not file.startswith('_'): module = file[:file.find('.py')]
# Copyright (c) Facebook, Inc. and its affiliates. # # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. import importlib import os from fairseq import registry from fairseq.optim.fairseq_optimizer import FairseqOptimizer from fairseq.optim.fp16_optimizer import FP16Optimizer, MemoryEfficientFP16Optimizer from fairseq.optim.bmuf import FairseqBMUF # noqa __all__ = [ "FairseqOptimizer", "FP16Optimizer", "MemoryEfficientFP16Optimizer", ] build_optimizer, register_optimizer, OPTIMIZER_REGISTRY = registry.setup_registry( "--optimizer", base_class=FairseqOptimizer, default="nag", ) # automatically import any Python files in the optim/ directory for file in os.listdir(os.path.dirname(__file__)): if file.endswith(".py") and not file.startswith("_"): module = file[:file.find(".py")] importlib.import_module("fairseq.optim." + module)
# Copyright (c) Facebook, Inc. and its affiliates. # # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. import importlib import os from fairseq import registry (build_monotonic_attention, register_monotonic_attention, MONOTONIC_ATTENTION_REGISTRY, _) = registry.setup_registry('--simul-type') for file in os.listdir(os.path.dirname(__file__)): if file.endswith('.py') and not file.startswith('_'): model_name = file[:file.find('.py')] importlib.import_module('examples.simultaneous_translation.modules.' + model_name)
# Copyright (c) Facebook, Inc. and its affiliates. # # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. import importlib import os from fairseq import registry _build_scoring, register_scoring, SCORING_REGISTRY = registry.setup_registry( "--scoring", default="bleu") def build_scorer(args, tgt_dict): from fairseq import utils if args.sacrebleu: utils.deprecation_warning( "--sacrebleu is deprecated. Please use --scoring sacrebleu instead." ) args.scoring = "sacrebleu" if args.scoring == "bleu": from fairseq.scoring import bleu return bleu.Scorer(tgt_dict.pad(), tgt_dict.eos(), tgt_dict.unk()) else: return _build_scoring(args) # automatically import any Python files in the current directory for file in os.listdir(os.path.dirname(__file__)):
# Copyright (c) Facebook, Inc. and its affiliates. # # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. import importlib import os from fairseq import registry build_agent, register_agent, MONOTONIC_AGENT = registry.setup_registry( '--agent-type') DEFAULT_EOS = '</s>' GET = 0 SEND = 1 for file in os.listdir(os.path.dirname(__file__)): if file.endswith('.py') and not file.startswith('_'): module = file[:file.find('.py')] importlib.import_module('agents.' + module)
from omegaconf import DictConfig __all__ = [ "FairseqOptimizer", "FP16Optimizer", "MemoryEfficientFP16Optimizer", "shard_", ] ( _build_optimizer, register_optimizer, OPTIMIZER_REGISTRY, OPTIMIZER_DATACLASS_REGISTRY, ) = registry.setup_registry("--optimizer", base_class=FairseqOptimizer, required=True) def build_optimizer(optimizer_cfg: Union[DictConfig, Namespace], params, *extra_args, **extra_kwargs): if all(isinstance(p, dict) for p in params): params = [t for p in params for t in p.values()] params = list(filter(lambda p: p.requires_grad, params)) return _build_optimizer(optimizer_cfg, params, *extra_args, **extra_kwargs) # automatically import any Python files in the optim/ directory for file in os.listdir(os.path.dirname(__file__)): if file.endswith(".py") and not file.startswith("_"): file_name = file[:file.find(".py")]
# Copyright (c) Facebook, Inc. and its affiliates. # # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. import importlib import os from fairseq import registry build_tokenizer, register_tokenizer, TOKENIZER_REGISTRY, _ = registry.setup_registry( "--tokenizer", default=None, ) build_bpe, register_bpe, BPE_REGISTRY, _ = registry.setup_registry( "--bpe", default=None, ) # automatically import any Python files in the encoders/ directory for file in sorted(os.listdir(os.path.dirname(__file__))): if file.endswith(".py") and not file.startswith("_"): module = file[: file.find(".py")] importlib.import_module("fairseq.data.encoders." + module)
# Copyright (c) Facebook, Inc. and its affiliates. # # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. import importlib import os from fairseq import registry (build_scorer, register_scorer, SCORER_REGISTRIES, _) = registry.setup_registry("--scorer-type") for file in os.listdir(os.path.dirname(__file__)): if file.endswith(".py") and not file.startswith("_"): module = file[:file.find(".py")] importlib.import_module("scorers." + module)
"""isort:skip_file""" import importlib import os from fairseq import registry from fairseq.criterions.fairseq_criterion import ( # noqa FairseqCriterion, LegacyFairseqCriterion, ) from omegaconf import DictConfig ( build_criterion_, register_criterion, CRITERION_REGISTRY, CRITERION_DATACLASS_REGISTRY, ) = registry.setup_registry("--criterion", base_class=FairseqCriterion, default="cross_entropy") def build_criterion(cfg: DictConfig, task): return build_criterion_(cfg, task) # automatically import any Python files in the criterions/ directory for file in sorted(os.listdir(os.path.dirname(__file__))): if file.endswith(".py") and not file.startswith("_"): file_name = file[:file.find(".py")] importlib.import_module("fairseq.criterions." + file_name)
def add_string(self, ref, pred): self.ref.append(ref) self.pred.append(pred) @abstractmethod def score(self) -> float: pass @abstractmethod def result_string(self) -> str: pass _build_scorer, register_scorer, SCORER_REGISTRY, _ = registry.setup_registry( "--scoring", default="bleu" ) def build_scorer(choice, tgt_dict): _choice = choice._name if isinstance(choice, DictConfig) else choice if _choice == "bleu": from fairseq.scoring import bleu return bleu.Scorer( bleu.BleuConfig(pad=tgt_dict.pad(), eos=tgt_dict.eos(), unk=tgt_dict.unk()) ) return _build_scorer(choice)
""" Scores references and hypos """ from abc import abstractmethod import logging from typing import * from fairseq import registry, metrics from fairseq.utils import item import numpy as np logger = logging.getLogger(__name__) build_eval_scorer, register_eval_scorer, EVAL_SCORER_REGISTRY = registry.setup_registry( '--eval-scorer', default=None, ) def add_eval_scoring_args(parser): parser.add_argument('--eval-scorer-args', type=str, metavar='JSON', help='generation args for validation inference, ' 'e.g., \'{"beam": 4, "lenpen": 0.6}\'') parser.add_argument( '--eval-scorer-detok', type=str, default="space", help='detokenizer before computing any scores (e.g., "moses"); ' 'required if using --eval-bleu; use "space" to '