def ensemble_with(self, others, ensemble_encoder_config=None, ensemble_decoder_config=None, decoder_kwargs={}, encoder_kwargs={}): config = get_config(self) if ensemble_decoder_config is None: ensemble_decoder_config = {"module": self.decoder.ENSEMBLE} elif isinstance(ensemble_decoder_config, str): ensemble_decoder_config = {"module": ensemble_decoder_config} if ensemble_encoder_config is None: ensemble_encoder_config = {"module": self.encoder.ENSEMBLE} elif isinstance(ensemble_encoder_config, str): ensemble_encoder_config = {"module": ensemble_encoder_config} config['preprocessor'] = self.preprocessor config['encoder'] = get_instance({ **ensemble_encoder_config, "models": ([self.encoder, *(other.encoder for other in others)]), **encoder_kwargs }) if ensemble_encoder_config is not False else self.encoder config['decoder'] = get_instance({ **ensemble_decoder_config, "models": ([self.decoder, *(other.decoder for other in others)]), **decoder_kwargs }) return get_instance(config)
def __init__(self, models, ensemble_span_scorer_module=None): super().__init__() self.contextualizers = torch.nn.ModuleList( [get_instance(m.contextualizer) for m in models]) self.n_labels = models[0].n_labels self.filter_predictions = models[0].filter_predictions if ensemble_span_scorer_module is None: ensemble_span_scorer_module = models[0].span_scorer.ENSEMBLE self.span_scorer = get_instance({ "module": ensemble_span_scorer_module, "models": [get_instance(m.span_scorer) for m in models] })
def init_modules(self): # Init modules that depend on the vocabulary with fork_rng(self.seed): with fork_rng(True): self.encoder = get_instance({ **self.encoder, "_preprocessor": self.preprocessor }) if not isinstance(self.encoder, torch.nn.Module) else self.encoder with fork_rng(True): self.decoder = get_instance( { **self.decoder, "_preprocessor": self.preprocessor, "_encoder": self.encoder }) if not isinstance(self.decoder, torch.nn.Module) else self.decoder
def __init__( self, preprocessor, encoder, decoder, seed=42, data_seed=None, batch_size=24, fast_lr=1.5e-3, main_lr=1.5e-3, bert_lr=4e-5, gradient_clip_val=5., metrics=None, warmup_rate=0.1, use_lr_schedules=True, dynamic_preprocessing=False, optimizer_cls=torch.optim.AdamW, _size_factor=20, _predict_kwargs={}, additional_hparams={}, ): """ :param preprocessor: dict Preprocessor module parameters :param encoder: dict or list of dict Word encoders module parameters :param decoder: dict Decoder module parameters :param seed: int Seed for the model weights :param data_seed: int Seed for the data shuffling :param batch_size: int Batch size :param fast_lr: float Top modules parameters' learning rate, typically higher than other parameters learning rates :param main_lr: float Intermediate modules parameters' learning rate :param bert_lr: float BERT modules parameters' learning rate :param gradient_clip_val: Use gradient clipping :param warmup_rate: float Apply warmup for how much of the training (defaults to 0.1 = 10%) :param use_lr_schedules: bool Use learning rate schedules :param optimizer_cls: str or type Torch optimizer class to use """ super().__init__() # monkey_patch() self.automatic_optimization = False if data_seed is None: data_seed = seed self.seed = seed self.data_seed = data_seed self.size_factor = _size_factor self.gradient_clip_val = gradient_clip_val self.fast_lr = fast_lr self.main_lr = main_lr self.bert_lr = bert_lr self.use_lr_schedules = use_lr_schedules self.warmup_rate = warmup_rate self.batch_size = batch_size self.optimizer_cls = getattr( import_module(optimizer_cls.rsplit(".", 1)[0]), optimizer_cls.rsplit(".", 1)[1]) if isinstance( optimizer_cls, str) else optimizer_cls self.dynamic_preprocessing = dynamic_preprocessing self.preprocessor = get_instance(preprocessor) self.additional_hparams = additional_hparams if metrics is None: metrics = { "exact": dict(module="dem", binarize_tag_threshold=1., binarize_label_threshold=1.), "approx": dict(module="dem", binarize_tag_threshold=False, binarize_label_threshold=1.), } self.metrics = MetricsCollection( {k: get_instance(m) for k, m in metrics.items()}) # Init postponed to setup self.encoder = encoder self.decoder = decoder if not any(voc.training for voc in self.preprocessor.vocabularies.values()): self.init_modules() self._time = time.time() self._predict_kwargs = _predict_kwargs
def load_pretrained(path, map_location=None): loaded = torch.load(path, map_location=map_location) instance = get_instance(loaded["config"]) instance.load_state_dict(loaded["state_dict"], strict=False) instance.eval() return instance
def __init__(self, models): super().__init__() self.models = torch.nn.ModuleList([get_instance(e) for e in models]) self._output_size = models[0].output_size