def load(self, fname=None):
if fname is not None:
self.load_path = fname
if self.pretrained_bert:
log.info(f"From pretrained {self.pretrained_bert}.")
config = AutoConfig.from_pretrained(self.pretrained_bert,
num_labels=self.n_classes,
output_attentions=False,
output_hidden_states=False)
self.model = AutoModelForSequenceClassification.from_pretrained(
self.pretrained_bert, config=config)
elif self.bert_config_file and Path(self.bert_config_file).is_file():
self.bert_config = AutoConfig.from_json_file(
str(expand_path(self.bert_config_file)))
if self.attention_probs_keep_prob is not None:
self.bert_config.attention_probs_dropout_prob = 1.0 - self.attention_probs_keep_prob
if self.hidden_keep_prob is not None:
self.bert_config.hidden_dropout_prob = 1.0 - self.hidden_keep_prob
self.model = AutoModelForSequenceClassification.from_config(
config=self.bert_config)
else:
raise ConfigError("No pre-trained BERT model is given.")
self.model.to(self.device)
self.optimizer = getattr(torch.optim, self.optimizer_name)(
self.model.parameters(), **self.optimizer_parameters)
if self.lr_scheduler_name is not None:
self.lr_scheduler = getattr(torch.optim.lr_scheduler,
self.lr_scheduler_name)(
self.optimizer,
**self.lr_scheduler_parameters)
if self.load_path:
log.info(f"Load path {self.load_path} is given.")
if isinstance(self.load_path,
Path) and not self.load_path.parent.is_dir():
raise ConfigError("Provided load path is incorrect!")
weights_path = Path(self.load_path.resolve())
weights_path = weights_path.with_suffix(f".pth.tar")
if weights_path.exists():
log.info(f"Load path {weights_path} exists.")
log.info(
f"Initializing `{self.__class__.__name__}` from saved.")
# now load the weights, optimizer from saved
log.info(f"Loading weights from {weights_path}.")
checkpoint = torch.load(weights_path, map_location=self.device)
self.model.load_state_dict(checkpoint["model_state_dict"])
self.optimizer.load_state_dict(
checkpoint["optimizer_state_dict"])
self.epochs_done = checkpoint.get("epochs_done", 0)
else:
log.info(
f"Init from scratch. Load path {weights_path} does not exist."
)
def load(self) -> None:
if self.pretrained_bert:
log.info(f"From pretrained {self.pretrained_bert}.")
self.config = AutoConfig.from_pretrained(
self.pretrained_bert, num_labels=self.n_classes, output_attentions=True, output_hidden_states=True
)
self.model = BertModel.from_pretrained(self.pretrained_bert, config=self.config)
elif self.bert_config_file and Path(self.bert_config_file).is_file():
self.config = AutoConfig.from_json_file(str(expand_path(self.bert_config_file)))
self.model = BertModel.from_config(config=self.bert_config)
else:
raise ConfigError("No pre-trained BERT model is given.")
self.model.to(self.device)
def load(self) -> None:
if self.pretrained_bert:
log.info(f"From pretrained {self.pretrained_bert}.")
self.pretrained_bert = str(expand_path(self.pretrained_bert))
self.config = AutoConfig.from_pretrained(self.pretrained_bert,
output_hidden_states=True)
self.encoder = AutoModel.from_pretrained(self.pretrained_bert,
config=self.config)
elif self.bert_config_file and Path(self.bert_config_file).is_file():
self.config = AutoConfig.from_json_file(
str(expand_path(self.bert_config_file)))
self.encoder = AutoModel.from_config(config=self.bert_config)
else:
raise ConfigError("No pre-trained BERT model is given.")
self.encoder.to(self.device)
def load(self, fname=None):
if fname is not None:
self.load_path = fname
if self.pretrained_bert:
logger.info(f"From pretrained {self.pretrained_bert}.")
config = AutoConfig.from_pretrained(self.pretrained_bert,
output_attentions=False,
output_hidden_states=False)
self.model = AutoModelForQuestionAnswering.from_pretrained(
self.pretrained_bert, config=config)
elif self.bert_config_file and Path(self.bert_config_file).is_file():
self.bert_config = AutoConfig.from_json_file(
str(expand_path(self.bert_config_file)))
if self.attention_probs_keep_prob is not None:
self.bert_config.attention_probs_dropout_prob = 1.0 - self.attention_probs_keep_prob
if self.hidden_keep_prob is not None:
self.bert_config.hidden_dropout_prob = 1.0 - self.hidden_keep_prob
self.model = AutoModelForQuestionAnswering(config=self.bert_config)
else:
raise ConfigError("No pre-trained BERT model is given.")
if self.device.type == "cuda" and torch.cuda.device_count() > 1:
self.model = torch.nn.DataParallel(self.model)
self.model.to(self.device)
self.optimizer = getattr(torch.optim, self.optimizer_name)(
self.model.parameters(), **self.optimizer_parameters)
if self.lr_scheduler_name is not None:
self.lr_scheduler = getattr(torch.optim.lr_scheduler,
self.lr_scheduler_name)(
self.optimizer,
**self.lr_scheduler_parameters)
if self.load_path:
logger.info(f"Load path {self.load_path} is given.")
if isinstance(self.load_path,
Path) and not self.load_path.parent.is_dir():
raise ConfigError("Provided load path is incorrect!")
weights_path = Path(self.load_path.resolve())
weights_path = weights_path.with_suffix(f".pth.tar")
if weights_path.exists():
logger.info(f"Load path {weights_path} exists.")
logger.info(
f"Initializing `{self.__class__.__name__}` from saved.")
# now load the weights, optimizer from saved
logger.info(f"Loading weights from {weights_path}.")
checkpoint = torch.load(weights_path, map_location=self.device)
model_state = checkpoint["model_state_dict"]
optimizer_state = checkpoint["optimizer_state_dict"]
# load a multi-gpu model on a single device
if not self.is_data_parallel and "module." in list(
model_state.keys())[0]:
tmp_model_state = {}
for key, value in model_state.items():
tmp_model_state[re.sub("module.", "", key)] = value
model_state = tmp_model_state
strict_load_flag = bool([
key for key in checkpoint["model_state_dict"].keys()
if key.endswith("embeddings.position_ids")
])
self.model.load_state_dict(model_state,
strict=strict_load_flag)
self.optimizer.load_state_dict(optimizer_state)
self.epochs_done = checkpoint.get("epochs_done", 0)
else:
logger.info(
f"Init from scratch. Load path {weights_path} does not exist."
)
def load(self, fname=None):
if fname is not None:
self.load_path = fname
if self.pretrained_bert:
log.info(f"From pretrained {self.pretrained_bert}.")
config = AutoConfig.from_pretrained(
self.pretrained_bert,
# num_labels=self.n_classes,
output_attentions=False,
output_hidden_states=False)
self.model = AutoModelForSequenceClassification.from_pretrained(
self.pretrained_bert, config=config)
try:
hidden_size = self.model.classifier.out_proj.in_features
if self.n_classes != self.model.num_labels:
self.model.classifier.out_proj.weight = torch.nn.Parameter(
torch.randn(self.n_classes, hidden_size))
self.model.classifier.out_proj.bias = torch.nn.Parameter(
torch.randn(self.n_classes))
self.model.classifier.out_proj.out_features = self.n_classes
self.model.num_labels = self.n_classes
except torch.nn.modules.module.ModuleAttributeError:
hidden_size = self.model.classifier.in_features
if self.n_classes != self.model.num_labels:
self.model.classifier.weight = torch.nn.Parameter(
torch.randn(self.n_classes, hidden_size))
self.model.classifier.bias = torch.nn.Parameter(
torch.randn(self.n_classes))
self.model.classifier.out_features = self.n_classes
self.model.num_labels = self.n_classes
elif self.bert_config_file and Path(self.bert_config_file).is_file():
self.bert_config = AutoConfig.from_json_file(
str(expand_path(self.bert_config_file)))
if self.attention_probs_keep_prob is not None:
self.bert_config.attention_probs_dropout_prob = 1.0 - self.attention_probs_keep_prob
if self.hidden_keep_prob is not None:
self.bert_config.hidden_dropout_prob = 1.0 - self.hidden_keep_prob
self.model = AutoModelForSequenceClassification.from_config(
config=self.bert_config)
else:
raise ConfigError("No pre-trained BERT model is given.")
self.model.to(self.device)
self.optimizer = getattr(torch.optim, self.optimizer_name)(
self.model.parameters(), **self.optimizer_parameters)
if self.lr_scheduler_name is not None:
self.lr_scheduler = getattr(torch.optim.lr_scheduler,
self.lr_scheduler_name)(
self.optimizer,
**self.lr_scheduler_parameters)
if self.load_path:
log.info(f"Load path {self.load_path} is given.")
if isinstance(self.load_path,
Path) and not self.load_path.parent.is_dir():
raise ConfigError("Provided load path is incorrect!")
weights_path = Path(self.load_path.resolve())
weights_path = weights_path.with_suffix(f".pth.tar")
if weights_path.exists():
log.info(f"Load path {weights_path} exists.")
log.info(
f"Initializing `{self.__class__.__name__}` from saved.")
# now load the weights, optimizer from saved
log.info(f"Loading weights from {weights_path}.")
checkpoint = torch.load(weights_path, map_location=self.device)
# set strict flag to False if position_ids are missing
# this is needed to load models trained on older versions
# of transformers library
strict_load_flag = bool([
key for key in checkpoint["model_state_dict"].keys()
if key.endswith("embeddings.position_ids")
])
self.model.load_state_dict(checkpoint["model_state_dict"],
strict=strict_load_flag)
self.optimizer.load_state_dict(
checkpoint["optimizer_state_dict"])
self.epochs_done = checkpoint.get("epochs_done", 0)
else:
log.info(
f"Init from scratch. Load path {weights_path} does not exist."
)
