def load(self, fname=None): if fname is not None: self.load_path = fname if self.pretrained_bert: log.info(f"From pretrained {self.pretrained_bert}.") config = AutoConfig.from_pretrained(self.pretrained_bert, num_labels=self.n_classes, output_attentions=False, output_hidden_states=False) self.model = AutoModelForSequenceClassification.from_pretrained( self.pretrained_bert, config=config) elif self.bert_config_file and Path(self.bert_config_file).is_file(): self.bert_config = AutoConfig.from_json_file( str(expand_path(self.bert_config_file))) if self.attention_probs_keep_prob is not None: self.bert_config.attention_probs_dropout_prob = 1.0 - self.attention_probs_keep_prob if self.hidden_keep_prob is not None: self.bert_config.hidden_dropout_prob = 1.0 - self.hidden_keep_prob self.model = AutoModelForSequenceClassification.from_config( config=self.bert_config) else: raise ConfigError("No pre-trained BERT model is given.") self.model.to(self.device) self.optimizer = getattr(torch.optim, self.optimizer_name)( self.model.parameters(), **self.optimizer_parameters) if self.lr_scheduler_name is not None: self.lr_scheduler = getattr(torch.optim.lr_scheduler, self.lr_scheduler_name)( self.optimizer, **self.lr_scheduler_parameters) if self.load_path: log.info(f"Load path {self.load_path} is given.") if isinstance(self.load_path, Path) and not self.load_path.parent.is_dir(): raise ConfigError("Provided load path is incorrect!") weights_path = Path(self.load_path.resolve()) weights_path = weights_path.with_suffix(f".pth.tar") if weights_path.exists(): log.info(f"Load path {weights_path} exists.") log.info( f"Initializing `{self.__class__.__name__}` from saved.") # now load the weights, optimizer from saved log.info(f"Loading weights from {weights_path}.") checkpoint = torch.load(weights_path, map_location=self.device) self.model.load_state_dict(checkpoint["model_state_dict"]) self.optimizer.load_state_dict( checkpoint["optimizer_state_dict"]) self.epochs_done = checkpoint.get("epochs_done", 0) else: log.info( f"Init from scratch. Load path {weights_path} does not exist." )
def load(self) -> None: if self.pretrained_bert: log.info(f"From pretrained {self.pretrained_bert}.") self.config = AutoConfig.from_pretrained( self.pretrained_bert, num_labels=self.n_classes, output_attentions=True, output_hidden_states=True ) self.model = BertModel.from_pretrained(self.pretrained_bert, config=self.config) elif self.bert_config_file and Path(self.bert_config_file).is_file(): self.config = AutoConfig.from_json_file(str(expand_path(self.bert_config_file))) self.model = BertModel.from_config(config=self.bert_config) else: raise ConfigError("No pre-trained BERT model is given.") self.model.to(self.device)
def load(self) -> None: if self.pretrained_bert: log.info(f"From pretrained {self.pretrained_bert}.") self.pretrained_bert = str(expand_path(self.pretrained_bert)) self.config = AutoConfig.from_pretrained(self.pretrained_bert, output_hidden_states=True) self.encoder = AutoModel.from_pretrained(self.pretrained_bert, config=self.config) elif self.bert_config_file and Path(self.bert_config_file).is_file(): self.config = AutoConfig.from_json_file( str(expand_path(self.bert_config_file))) self.encoder = AutoModel.from_config(config=self.bert_config) else: raise ConfigError("No pre-trained BERT model is given.") self.encoder.to(self.device)
def load(self, fname=None): if fname is not None: self.load_path = fname if self.pretrained_bert: logger.info(f"From pretrained {self.pretrained_bert}.") config = AutoConfig.from_pretrained(self.pretrained_bert, output_attentions=False, output_hidden_states=False) self.model = AutoModelForQuestionAnswering.from_pretrained( self.pretrained_bert, config=config) elif self.bert_config_file and Path(self.bert_config_file).is_file(): self.bert_config = AutoConfig.from_json_file( str(expand_path(self.bert_config_file))) if self.attention_probs_keep_prob is not None: self.bert_config.attention_probs_dropout_prob = 1.0 - self.attention_probs_keep_prob if self.hidden_keep_prob is not None: self.bert_config.hidden_dropout_prob = 1.0 - self.hidden_keep_prob self.model = AutoModelForQuestionAnswering(config=self.bert_config) else: raise ConfigError("No pre-trained BERT model is given.") if self.device.type == "cuda" and torch.cuda.device_count() > 1: self.model = torch.nn.DataParallel(self.model) self.model.to(self.device) self.optimizer = getattr(torch.optim, self.optimizer_name)( self.model.parameters(), **self.optimizer_parameters) if self.lr_scheduler_name is not None: self.lr_scheduler = getattr(torch.optim.lr_scheduler, self.lr_scheduler_name)( self.optimizer, **self.lr_scheduler_parameters) if self.load_path: logger.info(f"Load path {self.load_path} is given.") if isinstance(self.load_path, Path) and not self.load_path.parent.is_dir(): raise ConfigError("Provided load path is incorrect!") weights_path = Path(self.load_path.resolve()) weights_path = weights_path.with_suffix(f".pth.tar") if weights_path.exists(): logger.info(f"Load path {weights_path} exists.") logger.info( f"Initializing `{self.__class__.__name__}` from saved.") # now load the weights, optimizer from saved logger.info(f"Loading weights from {weights_path}.") checkpoint = torch.load(weights_path, map_location=self.device) model_state = checkpoint["model_state_dict"] optimizer_state = checkpoint["optimizer_state_dict"] # load a multi-gpu model on a single device if not self.is_data_parallel and "module." in list( model_state.keys())[0]: tmp_model_state = {} for key, value in model_state.items(): tmp_model_state[re.sub("module.", "", key)] = value model_state = tmp_model_state strict_load_flag = bool([ key for key in checkpoint["model_state_dict"].keys() if key.endswith("embeddings.position_ids") ]) self.model.load_state_dict(model_state, strict=strict_load_flag) self.optimizer.load_state_dict(optimizer_state) self.epochs_done = checkpoint.get("epochs_done", 0) else: logger.info( f"Init from scratch. Load path {weights_path} does not exist." )
def load(self, fname=None): if fname is not None: self.load_path = fname if self.pretrained_bert: log.info(f"From pretrained {self.pretrained_bert}.") config = AutoConfig.from_pretrained( self.pretrained_bert, # num_labels=self.n_classes, output_attentions=False, output_hidden_states=False) self.model = AutoModelForSequenceClassification.from_pretrained( self.pretrained_bert, config=config) try: hidden_size = self.model.classifier.out_proj.in_features if self.n_classes != self.model.num_labels: self.model.classifier.out_proj.weight = torch.nn.Parameter( torch.randn(self.n_classes, hidden_size)) self.model.classifier.out_proj.bias = torch.nn.Parameter( torch.randn(self.n_classes)) self.model.classifier.out_proj.out_features = self.n_classes self.model.num_labels = self.n_classes except torch.nn.modules.module.ModuleAttributeError: hidden_size = self.model.classifier.in_features if self.n_classes != self.model.num_labels: self.model.classifier.weight = torch.nn.Parameter( torch.randn(self.n_classes, hidden_size)) self.model.classifier.bias = torch.nn.Parameter( torch.randn(self.n_classes)) self.model.classifier.out_features = self.n_classes self.model.num_labels = self.n_classes elif self.bert_config_file and Path(self.bert_config_file).is_file(): self.bert_config = AutoConfig.from_json_file( str(expand_path(self.bert_config_file))) if self.attention_probs_keep_prob is not None: self.bert_config.attention_probs_dropout_prob = 1.0 - self.attention_probs_keep_prob if self.hidden_keep_prob is not None: self.bert_config.hidden_dropout_prob = 1.0 - self.hidden_keep_prob self.model = AutoModelForSequenceClassification.from_config( config=self.bert_config) else: raise ConfigError("No pre-trained BERT model is given.") self.model.to(self.device) self.optimizer = getattr(torch.optim, self.optimizer_name)( self.model.parameters(), **self.optimizer_parameters) if self.lr_scheduler_name is not None: self.lr_scheduler = getattr(torch.optim.lr_scheduler, self.lr_scheduler_name)( self.optimizer, **self.lr_scheduler_parameters) if self.load_path: log.info(f"Load path {self.load_path} is given.") if isinstance(self.load_path, Path) and not self.load_path.parent.is_dir(): raise ConfigError("Provided load path is incorrect!") weights_path = Path(self.load_path.resolve()) weights_path = weights_path.with_suffix(f".pth.tar") if weights_path.exists(): log.info(f"Load path {weights_path} exists.") log.info( f"Initializing `{self.__class__.__name__}` from saved.") # now load the weights, optimizer from saved log.info(f"Loading weights from {weights_path}.") checkpoint = torch.load(weights_path, map_location=self.device) # set strict flag to False if position_ids are missing # this is needed to load models trained on older versions # of transformers library strict_load_flag = bool([ key for key in checkpoint["model_state_dict"].keys() if key.endswith("embeddings.position_ids") ]) self.model.load_state_dict(checkpoint["model_state_dict"], strict=strict_load_flag) self.optimizer.load_state_dict( checkpoint["optimizer_state_dict"]) self.epochs_done = checkpoint.get("epochs_done", 0) else: log.info( f"Init from scratch. Load path {weights_path} does not exist." )