def ensemble_with(self,
others,
ensemble_encoder_config=None,
ensemble_decoder_config=None,
decoder_kwargs={},
encoder_kwargs={}):
config = get_config(self)
if ensemble_decoder_config is None:
ensemble_decoder_config = {"module": self.decoder.ENSEMBLE}
elif isinstance(ensemble_decoder_config, str):
ensemble_decoder_config = {"module": ensemble_decoder_config}
if ensemble_encoder_config is None:
ensemble_encoder_config = {"module": self.encoder.ENSEMBLE}
elif isinstance(ensemble_encoder_config, str):
ensemble_encoder_config = {"module": ensemble_encoder_config}
config['preprocessor'] = self.preprocessor
config['encoder'] = get_instance({
**ensemble_encoder_config, "models":
([self.encoder, *(other.encoder for other in others)]),
**encoder_kwargs
}) if ensemble_encoder_config is not False else self.encoder
config['decoder'] = get_instance({
**ensemble_decoder_config, "models":
([self.decoder, *(other.decoder for other in others)]),
**decoder_kwargs
})
return get_instance(config)
def __init__(self, models, ensemble_span_scorer_module=None):
super().__init__()
self.contextualizers = torch.nn.ModuleList(
[get_instance(m.contextualizer) for m in models])
self.n_labels = models[0].n_labels
self.filter_predictions = models[0].filter_predictions
if ensemble_span_scorer_module is None:
ensemble_span_scorer_module = models[0].span_scorer.ENSEMBLE
self.span_scorer = get_instance({
"module":
ensemble_span_scorer_module,
"models": [get_instance(m.span_scorer) for m in models]
})
def init_modules(self):
# Init modules that depend on the vocabulary
with fork_rng(self.seed):
with fork_rng(True):
self.encoder = get_instance({
**self.encoder, "_preprocessor":
self.preprocessor
}) if not isinstance(self.encoder,
torch.nn.Module) else self.encoder
with fork_rng(True):
self.decoder = get_instance(
{
**self.decoder, "_preprocessor": self.preprocessor,
"_encoder": self.encoder
}) if not isinstance(self.decoder,
torch.nn.Module) else self.decoder
def __init__(
self,
preprocessor,
encoder,
decoder,
seed=42,
data_seed=None,
batch_size=24,
fast_lr=1.5e-3,
main_lr=1.5e-3,
bert_lr=4e-5,
gradient_clip_val=5.,
metrics=None,
warmup_rate=0.1,
use_lr_schedules=True,
dynamic_preprocessing=False,
optimizer_cls=torch.optim.AdamW,
_size_factor=20,
_predict_kwargs={},
additional_hparams={},
):
"""
:param preprocessor: dict
Preprocessor module parameters
:param encoder: dict or list of dict
Word encoders module parameters
:param decoder: dict
Decoder module parameters
:param seed: int
Seed for the model weights
:param data_seed: int
Seed for the data shuffling
:param batch_size: int
Batch size
:param fast_lr: float
Top modules parameters' learning rate, typically higher than other parameters learning rates
:param main_lr: float
Intermediate modules parameters' learning rate
:param bert_lr: float
BERT modules parameters' learning rate
:param gradient_clip_val:
Use gradient clipping
:param warmup_rate: float
Apply warmup for how much of the training (defaults to 0.1 = 10%)
:param use_lr_schedules: bool
Use learning rate schedules
:param optimizer_cls: str or type
Torch optimizer class to use
"""
super().__init__()
# monkey_patch()
self.automatic_optimization = False
if data_seed is None:
data_seed = seed
self.seed = seed
self.data_seed = data_seed
self.size_factor = _size_factor
self.gradient_clip_val = gradient_clip_val
self.fast_lr = fast_lr
self.main_lr = main_lr
self.bert_lr = bert_lr
self.use_lr_schedules = use_lr_schedules
self.warmup_rate = warmup_rate
self.batch_size = batch_size
self.optimizer_cls = getattr(
import_module(optimizer_cls.rsplit(".", 1)[0]),
optimizer_cls.rsplit(".", 1)[1]) if isinstance(
optimizer_cls, str) else optimizer_cls
self.dynamic_preprocessing = dynamic_preprocessing
self.preprocessor = get_instance(preprocessor)
self.additional_hparams = additional_hparams
if metrics is None:
metrics = {
"exact":
dict(module="dem",
binarize_tag_threshold=1.,
binarize_label_threshold=1.),
"approx":
dict(module="dem",
binarize_tag_threshold=False,
binarize_label_threshold=1.),
}
self.metrics = MetricsCollection(
{k: get_instance(m)
for k, m in metrics.items()})
# Init postponed to setup
self.encoder = encoder
self.decoder = decoder
if not any(voc.training
for voc in self.preprocessor.vocabularies.values()):
self.init_modules()
self._time = time.time()
self._predict_kwargs = _predict_kwargs
def load_pretrained(path, map_location=None):
loaded = torch.load(path, map_location=map_location)
instance = get_instance(loaded["config"])
instance.load_state_dict(loaded["state_dict"], strict=False)
instance.eval()
return instance
def __init__(self, models):
super().__init__()
self.models = torch.nn.ModuleList([get_instance(e) for e in models])
self._output_size = models[0].output_size