def test_local_and_two_global_namespace_plugins(self):
available_plugins = set(discover_plugins())
self.assertSetEqual(set(), available_plugins)
# We make plugins "a" and "c" available as packages, each from other directories, as if they were
# separate installed projects ("global" usage of the plugins).
# We move to another directory with a different plugin "b", as if it were another separate project
# which is not installed ("local" usage of the plugin declared in the namespace).
with pip_install(self.project_a_fixtures_root, "a"), pip_install(
self.project_c_fixtures_root,
"c"), push_python_project(self.project_b_fixtures_root):
available_plugins = set(discover_plugins())
self.assertSetEqual(
{
"allennlp_plugins.a", "allennlp_plugins.b",
"allennlp_plugins.c"
},
available_plugins,
)
import_plugins()
subcommands_available = Subcommand.list_available()
self.assertIn("a", subcommands_available)
self.assertIn("b", subcommands_available)
self.assertIn("c", subcommands_available)
def from_path(
cls,
archive_path: Union[str, PathLike],
predictor_name: str = None,
cuda_device: int = -1,
dataset_reader_to_load: str = "validation",
frozen: bool = True,
import_plugins: bool = True,
overrides: Union[str, Dict[str, Any]] = "",
**kwargs,
) -> "Predictor":
"""
Instantiate a `Predictor` from an archive path.
If you need more detailed configuration options, such as overrides,
please use `from_archive`.
# Parameters
archive_path : `Union[str, PathLike]`
The path to the archive.
predictor_name : `str`, optional (default=`None`)
Name that the predictor is registered as, or None to use the
predictor associated with the model.
cuda_device : `int`, optional (default=`-1`)
If `cuda_device` is >= 0, the model will be loaded onto the
corresponding GPU. Otherwise it will be loaded onto the CPU.
dataset_reader_to_load : `str`, optional (default=`"validation"`)
Which dataset reader to load from the archive, either "train" or
"validation".
frozen : `bool`, optional (default=`True`)
If we should call `model.eval()` when building the predictor.
import_plugins : `bool`, optional (default=`True`)
If `True`, we attempt to import plugins before loading the predictor.
This comes with additional overhead, but means you don't need to explicitly
import the modules that your predictor depends on as long as those modules
can be found by `allennlp.common.plugins.import_plugins()`.
overrides : `Union[str, Dict[str, Any]]`, optional (default = `""`)
JSON overrides to apply to the unarchived `Params` object.
**kwargs : `Any`
Additional key-word arguments that will be passed to the `Predictor`'s
`__init__()` method.
# Returns
`Predictor`
A Predictor instance.
"""
if import_plugins:
plugins.import_plugins()
return Predictor.from_archive(
load_archive(archive_path,
cuda_device=cuda_device,
overrides=overrides),
predictor_name,
dataset_reader_to_load=dataset_reader_to_load,
frozen=frozen,
extra_args=kwargs,
)
def setup_class(cls):
# Make sure all the classes we need are registered.
import_plugins()
# Patch dataset paths.
for dataset_patch, patch_targets in DATASET_PATCHES.items():
for patch_target in patch_targets:
os.environ[patch_target] = str(dataset_patch)
def from_path(
cls,
archive_path: Union[str, PathLike],
*,
interpreter_name: Optional[str] = None,
train_data_path: Optional[DatasetReaderInput] = None,
train_data_loader: Lazy[DataLoader] = Lazy(
SimpleDataLoader.from_dataset_reader),
test_data_loader: Lazy[DataLoader] = Lazy(
SimpleDataLoader.from_dataset_reader),
params_to_freeze: Optional[List[str]] = None,
cuda_device: int = -1,
import_plugins: bool = True,
overrides: Union[str, Dict[str, Any]] = "",
**extras,
) -> "InfluenceInterpreter":
"""
Load an `InfluenceInterpreter` from an archive path.
# Parameters
archive_path : `Union[str, PathLike]`, required
The path to the archive file.
interpreter_name : `Optional[str]`, optional (default = `None`)
The registered name of the an interpreter class. If not specified,
the default implementation (`SimpleInfluence`) will be used.
train_data_path : `Optional[DatasetReaderInput]`, optional (default = `None`)
If not specified, `train_data_path` will be taken from the archive's config.
train_data_loader : `Lazy[DataLoader]`, optional (default = `Lazy(SimpleDataLoader)`)
test_data_loader : `Lazy[DataLoader]`, optional (default = `Lazy(SimpleDataLoader)`)
params_to_freeze : `Optional[List[str]]`, optional (default = `None`)
cuda_device : `int`, optional (default = `-1`)
import_plugins : `bool`, optional (default = `True`)
If `True`, we attempt to import plugins before loading the `InfluenceInterpreter`.
This comes with additional overhead, but means you don't need to explicitly
import the modules that your implementation depends on as long as those modules
can be found by `allennlp.common.plugins.import_plugins()`.
overrides : `Union[str, Dict[str, Any]]`, optional (default = `""`)
JSON overrides to apply to the unarchived `Params` object.
**extras : `Any`
Extra parameters to pass to the interpreter's `__init__()` method.
"""
if import_plugins:
plugins.import_plugins()
return cls.from_archive(
load_archive(archive_path,
cuda_device=cuda_device,
overrides=overrides),
interpreter_name=interpreter_name,
train_data_path=train_data_path,
train_data_loader=train_data_loader,
test_data_loader=test_data_loader,
params_to_freeze=params_to_freeze,
cuda_device=cuda_device,
**extras,
)
def test_file_plugin(self):
available_plugins = set(discover_plugins())
self.assertSetEqual(set(), available_plugins)
with pushd(self.project_d_fixtures_root):
available_plugins = set(discover_plugins())
self.assertSetEqual({"d"}, available_plugins)
import_plugins()
subcommands_available = Subcommand.list_available()
self.assertIn("d", subcommands_available)
def test_local_namespace_plugin(self):
available_plugins = set(discover_plugins())
self.assertSetEqual(set(), available_plugins)
with pushd(self.project_b_fixtures_root):
available_plugins = set(discover_plugins())
self.assertSetEqual({"allennlp_plugins.b"}, available_plugins)
import_plugins()
subcommands_available = Subcommand.list_available()
self.assertIn("b", subcommands_available)
def test_global_namespace_plugin(self):
available_plugins = set(discover_plugins())
self.assertSetEqual(set(), available_plugins)
with pip_install(self.project_a_fixtures_root, "a"):
available_plugins = set(discover_plugins())
self.assertSetEqual({"allennlp_plugins.a"}, available_plugins)
import_plugins()
subcommands_available = Subcommand.list_available()
self.assertIn("a", subcommands_available)
def test_file_plugin(self):
available_plugins = set(discover_plugins())
assert available_plugins == set()
with pushd(self.plugins_root):
available_plugins = set(discover_plugins())
assert available_plugins == {"d"}
import_plugins()
subcommands_available = Subcommand.list_available()
assert "d" in subcommands_available
def parse_args(
prog: Optional[str] = None
) -> Tuple[argparse.ArgumentParser, argparse.Namespace]:
"""
Creates the argument parser for the main program and uses it to parse the args.
"""
parser = ArgumentParserWithDefaults(description="Run AllenNLP", prog=prog)
parser.add_argument("--version",
action="version",
version=f"%(prog)s {__version__}")
subparsers = parser.add_subparsers(title="Commands", metavar="")
subcommands: Set[str] = set()
def add_subcommands():
for subcommand_name in sorted(Subcommand.list_available()):
if subcommand_name in subcommands:
continue
subcommands.add(subcommand_name)
subcommand_class = Subcommand.by_name(subcommand_name)
subcommand = subcommand_class()
subparser = subcommand.add_subparser(subparsers)
if subcommand_class.requires_plugins:
subparser.add_argument(
"--include-package",
type=str,
action="append",
default=[],
help="additional packages to include",
)
# Add all default registered subcommands first.
add_subcommands()
# If we need to print the usage/help, or the subcommand is unknown,
# we'll call `import_plugins()` to register any plugin subcommands first.
argv = sys.argv[1:]
plugins_imported: bool = False
if not argv or argv == ["--help"] or argv[0] not in subcommands:
import_plugins()
plugins_imported = True
# Add subcommands again in case one of the plugins has a registered subcommand.
add_subcommands()
# Now we can parse the arguments.
args = parser.parse_args()
if not plugins_imported and Subcommand.by_name(
argv[0]).requires_plugins: # type: ignore
import_plugins()
return parser, args
def get_pretrained_models() -> Dict[str, ModelCard]:
"""
Returns a mapping of [`ModelCard`](/models/common/model_card#modelcard)s for all
available pretrained models.
"""
import_plugins()
pretrained_models = {}
model_card_paths = os.path.join(
os.path.dirname(os.path.realpath(__file__)), "modelcards", "*.json"
)
for model_card_path in glob.glob(model_card_paths):
if "template" not in model_card_path:
model_card = ModelCard.from_params(params=Params.from_file(model_card_path))
pretrained_models[model_card.id] = model_card
return pretrained_models
def test_local_namespace_plugin_different_path(self):
available_plugins = set(discover_plugins())
self.assertSetEqual(set(), available_plugins)
with tempfile.TemporaryDirectory() as temp_dir_b:
distutils.dir_util.copy_tree(self.project_b_fixtures_root, temp_dir_b)
# We move to another directory with a different plugin "b", as if it were another
# separate project which is not installed ("local" usage of the plugin declared in
# the namespace).
with pushd(temp_dir_b):
available_plugins = set(discover_plugins())
self.assertSetEqual({"allennlp_plugins.b"}, available_plugins)
import_plugins()
subcommands_available = Subcommand.list_available()
self.assertIn("b", subcommands_available)
def init_load_model():
"""
The [`run`](./train.md#run) command only knows about the registered classes in the ``allennlp``
codebase. In particular, once you start creating your own `Model` s and so forth, it won't
work for them, unless you use the ``--include-package`` flag or you make your code available
as a plugin (see [`plugins`](./plugins.md)).
"""
global predict_manager, cuda_place
import_plugins()
parser = create_parser("allennlp")
args = parser.parse_args()
args.archive_file = r"models/ace05_event/model.tar.gz"
args.cuda_device = cuda_place
args.input_file = r"data/ace-event/processed-data/default-settings/json/test.json"
args.output_file = r"predictions/event_predict.json"
args.use_dataset_reader = True
args.include_package = "dygie"
args.predictor = "dygie"
args.weights_file = None
args.overrides = ''
args.dataset_reader_choice = 'validation'
args.batch_size = 1
args.silent = False
# print(dir(args))
# print(args.include_package)
# for package_name in args.include_package:
# import_module_and_submodules_new(package_name)
import_module_and_submodules_new('dygie')
import_module_and_submodules_new('dygie.predictors.dygie')
import_module_and_submodules_new('dygie.models.dygie')
# predictor = DyGIEPredictor.from_path(os.path.join(current_path, 'models/ace05_event/model.tar.gz'), predictor_name="dygie")
predictor = predict._get_predictor(args)
predict_manager = predict._PredictManager(
predictor,
args.input_file,
args.output_file,
args.batch_size,
not args.silent,
args.use_dataset_reader,
)
def test_reload_plugins_adds_new(self):
available_plugins = set(discover_plugins())
self.assertSetEqual(set(), available_plugins)
with pip_install(self.project_a_fixtures_root, "a"):
available_plugins = set(discover_plugins())
self.assertSetEqual({"allennlp_plugins.a"}, available_plugins)
import_plugins()
subcommands_available = Subcommand.list_available()
self.assertIn("a", subcommands_available)
with pip_install(self.project_c_fixtures_root, "c"):
available_plugins = set(discover_plugins())
self.assertSetEqual({"allennlp_plugins.a", "allennlp_plugins.c"}, available_plugins)
import_plugins()
subcommands_available = Subcommand.list_available()
self.assertIn("a", subcommands_available)
self.assertIn("c", subcommands_available)
def main(prog: Optional[str] = None) -> None:
"""
The [`run`](./train.md#run) command only knows about the registered classes in the ``allennlp``
codebase. In particular, once you start creating your own `Model` s and so forth, it won't
work for them, unless you use the ``--include-package`` flag or you make your code available
as a plugin (see [`plugins`](./plugins.md)).
"""
import_plugins()
parser = create_parser(prog)
args = parser.parse_args()
# If a subparser is triggered, it adds its work as `args.func`.
# So if no such attribute has been added, no subparser was triggered,
# so give the user some help.
if "func" in dir(args):
# Import any additional modules needed (to register custom classes).
for package_name in args.include_package:
import_module_and_submodules(package_name)
args.func(args)
else:
parser.print_help()
def from_path(
cls,
archive_path: str,
predictor_name: str = None,
cuda_device: int = -1,
dataset_reader_to_load: str = "validation",
frozen: bool = True,
import_plugins: bool = True,
language: str = "en_core_web_sm",
restrict_frames: bool = False,
restrict_roles: bool = False,
) -> "Predictor":
if import_plugins:
plugins.import_plugins()
return SrlTransformersPredictor.from_archive(
load_archive(archive_path, cuda_device=cuda_device),
predictor_name,
dataset_reader_to_load=dataset_reader_to_load,
frozen=frozen,
language=language,
restrict_frames=restrict_frames,
restrict_roles=restrict_roles,
)
#!/usr/bin/env python
"""
Ensures models are automatically found by allennlp.
"""
import logging
from allennlp.common.plugins import import_plugins
from allennlp.models import Model
logging.basicConfig(level=logging.INFO)
import_plugins()
Model.by_name("copynet_seq2seq")
def _train_worker(
process_rank: int,
params: Params,
serialization_dir: str,
file_friendly_logging: bool = False,
include_package: List[str] = None,
batch_weight_key: str = "",
node_rank: int = 0,
master_addr: str = "127.0.0.1",
master_port: int = 29500,
world_size: int = 1,
distributed_device_ids: List[str] = None,
) -> Optional[Model]:
"""
Helper to train the configured model/experiment. In distributed mode, this is spawned as a
worker process. In a single GPU experiment, this returns the ``Model`` object and in distributed
training, nothing is returned.
# Parameters
process_rank : ``int``
The process index that is initialized using the GPU device id.
params : ``Params``
A parameter object specifying an AllenNLP Experiment.
serialization_dir : ``str``
The directory in which to save results and logs.
file_friendly_logging : ``bool``, optional (default=False)
If ``True``, we add newlines to tqdm output, even on an interactive terminal, and we slow
down tqdm's output to only once every 10 seconds.
include_package : ``List[str]``, optional
In distributed mode, since this function would have been spawned as a separate process,
the extra imports need to be done again. NOTE: This does not have any effect in single
GPU training.
batch_weight_key : ``str``, optional (default="")
If non-empty, name of metric used to weight the loss on a per-batch basis.
node_rank : ``int``, optional
Rank of the node.
master_addr : ``str``, optional (default="127.0.0.1")
Address of the master node for distributed training.
master_port : ``str``, optional (default="29500")
Port of the master node for distributed training.
world_size : ``int``, optional
The number of processes involved in distributed training.
distributed_device_ids: ``List[str]``, optional
IDs of the devices used involved in distributed training.
# Returns
best_model : ``Model``
The model with the best epoch weights.
"""
common_util.prepare_global_logging(serialization_dir,
file_friendly_logging,
rank=process_rank,
world_size=world_size)
common_util.prepare_environment(params)
distributed = world_size > 1
# not using `allennlp.common.util.is_master` as the process group is yet to be initialized
master = process_rank == 0
include_package = include_package or []
if distributed:
# Since the worker is spawned and not forked, the extra imports need to be done again.
import_plugins()
for package_name in include_package:
common_util.import_submodules(package_name)
num_procs_per_node = len(distributed_device_ids)
# The Unique identifier of the worker process among all the processes in the
# distributed training group is computed here. This is used while initializing
# the process group using `init_process_group`
global_rank = node_rank * num_procs_per_node + process_rank
# Number of processes per node is useful to know if a process
# is a master in the local node(node in which it is running)
os.environ["ALLENNLP_PROCS_PER_NODE"] = str(num_procs_per_node)
# In distributed training, the configured device is always going to be a list.
# The corresponding gpu id for the particular worker is obtained by picking the id
# from the device list with the rank as index
gpu_id = distributed_device_ids[process_rank] # type: ignore
# Till now, "cuda_device" might not be set in the trainer params.
# But a worker trainer needs to only know about its specific GPU id.
params["trainer"]["cuda_device"] = gpu_id
params["trainer"]["world_size"] = world_size
params["trainer"]["distributed"] = True
torch.cuda.set_device(int(gpu_id))
dist.init_process_group(
backend="nccl",
init_method=f"tcp://{master_addr}:{master_port}",
world_size=world_size,
rank=global_rank,
)
logging.info(f"Process group of world size {world_size} initialized "
f"for distributed training in worker {global_rank}")
train_loop = TrainModel.from_params(
params=params,
serialization_dir=serialization_dir,
local_rank=process_rank,
batch_weight_key=batch_weight_key,
)
try:
if distributed: # let the setup get ready for all the workers
dist.barrier()
metrics = train_loop.run()
except KeyboardInterrupt:
# if we have completed an epoch, try to create a model archive.
if master and os.path.exists(
os.path.join(serialization_dir, _DEFAULT_WEIGHTS)):
logging.info(
"Training interrupted by the user. Attempting to create "
"a model archive using the current best epoch weights.")
archive_model(serialization_dir)
raise
if master:
train_loop.finish(metrics)
if not distributed:
return train_loop.model
return None # to make mypy happy
def _train_worker(
process_rank: int,
params: Params,
serialization_dir: Union[str, PathLike],
include_package: List[str] = None,
dry_run: bool = False,
node_rank: int = 0,
primary_addr: str = "127.0.0.1",
primary_port: int = 29500,
world_size: int = 1,
distributed_device_ids: List[int] = None,
file_friendly_logging: bool = False,
include_in_archive: List[str] = None,
distributed_params: Optional[Params] = None,
) -> Optional[Model]:
"""
Helper to train the configured model/experiment. In distributed mode, this is spawned as a
worker process. In a single GPU experiment, this returns the `Model` object and in distributed
training, nothing is returned.
# Parameters
process_rank : `int`
The process index that is initialized using the GPU device id.
params : `Params`
A parameter object specifying an AllenNLP Experiment.
serialization_dir : `str`
The directory in which to save results and logs.
include_package : `List[str]`, optional
In distributed mode, since this function would have been spawned as a separate process,
the extra imports need to be done again. NOTE: This does not have any effect in single
GPU training.
dry_run : `bool`, optional (default=`False`)
Do not train a model, but create a vocabulary, show dataset statistics and other training
information.
node_rank : `int`, optional
Rank of the node.
primary_addr : `str`, optional (default=`"127.0.0.1"`)
Address of the primary node for distributed training.
primary_port : `str`, optional (default=`"29500"`)
Port of the primary node for distributed training.
world_size : `int`, optional
The number of processes involved in distributed training.
distributed_device_ids: `List[str]`, optional
IDs of the devices used involved in distributed training.
file_friendly_logging : `bool`, optional (default=`False`)
If `True`, we add newlines to tqdm output, even on an interactive terminal, and we slow
down tqdm's output to only once every 10 seconds.
include_in_archive : `List[str]`, optional
Paths relative to `serialization_dir` that should be archived in addition to the default ones.
distributed_params : `Optional[Params]`, optional
Additional distributed params.
# Returns
best_model : `Optional[Model]`
The model with the best epoch weights or `None` if in distributed training or in dry run.
"""
common_logging.FILE_FRIENDLY_LOGGING = file_friendly_logging
common_logging.prepare_global_logging(
serialization_dir,
rank=process_rank,
world_size=world_size,
)
common_util.prepare_environment(params)
distributed = world_size > 1
primary = process_rank == 0
include_package = include_package or []
ddp_accelerator: Optional[DdpAccelerator] = None
if distributed:
assert distributed_device_ids is not None
assert distributed_params is not None
# Since the worker is spawned and not forked, the extra imports need to be done again.
# Both the ones from the plugins and the ones from `include_package`.
import_plugins()
for package_name in include_package:
common_util.import_module_and_submodules(package_name)
num_procs_per_node = len(distributed_device_ids)
# The Unique identifier of the worker process among all the processes in the
# distributed training group is computed here. This is used while initializing
# the process group using `init_process_group`
global_rank = node_rank * num_procs_per_node + process_rank
# Number of processes per node is useful to know if a process
# is a primary in the local node(node in which it is running)
os.environ["ALLENNLP_PROCS_PER_NODE"] = str(num_procs_per_node)
# In distributed training, the configured device is always going to be a list.
# The corresponding gpu id for the particular worker is obtained by picking the id
# from the device list with the rank as index
gpu_id = int(distributed_device_ids[process_rank]) # type: ignore
# Till now, "cuda_device" might not be set in the trainer params.
# But a worker trainer needs to only know about its specific GPU id.
params["trainer"]["local_rank"] = process_rank
params["trainer"]["cuda_device"] = gpu_id
params["trainer"]["world_size"] = world_size
params["trainer"]["distributed"] = True
if gpu_id >= 0:
torch.cuda.set_device(gpu_id)
dist.init_process_group(
backend="nccl",
init_method=f"tcp://{primary_addr}:{primary_port}",
world_size=world_size,
rank=global_rank,
)
else:
dist.init_process_group(
backend="gloo",
init_method=f"tcp://{primary_addr}:{primary_port}",
world_size=world_size,
rank=global_rank,
)
if "ddp_accelerator" in distributed_params:
ddp_accelerator_params = distributed_params.pop("ddp_accelerator")
ddp_accelerator = DdpAccelerator.from_params(
ddp_accelerator_params,
local_rank=process_rank,
world_size=world_size,
cuda_device=gpu_id,
)
logging.info(f"Process group of world size {world_size} initialized "
f"for distributed training in worker {global_rank}")
train_loop = TrainModel.from_params(
params=params,
serialization_dir=serialization_dir,
local_rank=process_rank,
ddp_accelerator=ddp_accelerator,
)
if dry_run:
return None
try:
if distributed: # let the setup get ready for all the workers
dist.barrier()
metrics = train_loop.run()
except KeyboardInterrupt:
# if we have completed an epoch, try to create a model archive.
if primary:
best_weights_path = train_loop.trainer.get_best_weights_path()
if best_weights_path is None:
logging.info(
"Training interrupted by the user, and no best model has been saved. "
"No model archive created.")
else:
logging.info(
"Training interrupted by the user. Attempting to create "
"a model archive using the current best epoch weights.")
archive_model(
serialization_dir,
weights=best_weights_path,
include_in_archive=include_in_archive,
)
raise
if primary:
train_loop.finish(metrics)
if not distributed:
return train_loop.model
return None
