def tell_story(event: LambdaDict, context: LambdaDict) -> LambdaDict:
"""
Function to take string input, and use text generation models to create
a story which can serve as a natural language response to player inputs
:param event: Input AWS Lambda event dict
:param context: Input AWS Lambda context dict
:return: Output AWS Lambda dict
"""
# Decode the request
request_body = event.get("body")
if type(request_body) == str:
request_body = json.loads(request_body)
story_context = request_body["context"]
print(story_context)
# Load model
"""
tokenizer = AutoTokenizer.from_pretrained("microsoft/DialoGPT-small")
model = AutoModelWithLMHead.from_pretrained("microsoft/DialoGPT-small")
tokenizer = AutoTokenizer.from_pretrained("sshleifer/bart-tiny-random")
model = AutoModelWithLMHead.from_pretrained("sshleifer/bart-tiny-random")
"""
tokenizer = AutoTokenizer.from_pretrained("distilgpt2")
model = AutoModelWithLMHead.from_pretrained("distilgpt2")
# Compute story
# encode the new user input, add the eos_token and return a tensor in Pytorch
story_input = tokenizer.encode(story_context + tokenizer.eos_token,
return_tensors='pt')
# generated a response while limiting the total chat history to 1000 tokens,
generated_story = model.generate(story_input,
pad_token_id=tokenizer.eos_token_id)
# pretty print last ouput tokens from bot
story_result = tokenizer.decode(generated_story[:,
story_input.shape[-1]:][0],
skip_special_tokens=True)
result = {
"statusCode": 200,
"body": story_result,
"headers": {
"Access-Control-Allow-Origin": "*"
},
}
return result
def __init__(self,
pretrained_model_path="facebook/bart-large-cnn",
max_length=1024):
self.pretrained_model_path = pretrained_model_path
self.max_length = max_length
self.tokenizer = AutoTokenizer.from_pretrained(pretrained_model_path)
self.model = AutoModelWithLMHead.from_pretrained(pretrained_model_path)
task_specific_params = self.model.config.task_specific_params
if task_specific_params is not None:
self.model.config.update(
task_specific_params.get("summarization", {}))
def init_model(model_name: str, device, do_lower_case: bool = False):
"""
Initialize a pre-trained LM
:param model_name: from MODEL_CLASSES
:param device: CUDA / CPU device
:param do_lower_case: whether the model is lower cased or not
:return: the model and tokenizer
"""
tokenizer = AutoTokenizer.from_pretrained(model_name,
do_lower_case=do_lower_case)
model = AutoModelWithLMHead.from_pretrained(model_name)
model.to(device)
model.eval()
return tokenizer, model
def __init__(self, hparams):
super().__init__(hparams)
self.tokenizer = AutoTokenizer.from_pretrained(
"huggingface/CodeBERTa-small-v1", resume_download=True)
# FIXME: using mask lm head in pretraining, but using automodel in finetuning.
# self.lm_model = AutoModel.from_pretrained(
# "huggingface/CodeBERTa-small-v1", resume_download=True, config=hparams["roberta_config"])
self.model = AutoModelWithLMHead.from_config(
AutoConfig.from_pretrained("huggingface/CodeBERTa-small-v1",
resume_download=True,
config=hparams["roberta_config"]))
def __init__(self, segment_size, output_size, dropout, vocab_size):
super(ALBertSmallDenseHiddenPunc, self).__init__()
self.bert = AutoModelWithLMHead.from_pretrained(
'./models/albert_chinese_small/')
# self.bert_vocab_size = vocab_size
# self.bn = nn.BatchNorm1d(segment_size*self.bert_vocab_size)
# self.fc = nn.Linear(segment_size*self.bert_vocab_size, output_size)
self.albert = self.bert.albert
self.dense = self.bert.predictions.dense
# 批标准化
self.bn = nn.BatchNorm1d(segment_size * 128)
# NOTE rnn_hidden*2 使用bert中间层hidden_state 384
self.fc = nn.Linear(segment_size * 128, output_size)
self.dropout = nn.Dropout(dropout)
def __init__(self, params):
super(BertTagger, self).__init__()
self.num_tag = params.num_tag
self.hidden_dim = params.hidden_dim
config = AutoConfig.from_pretrained(params.model_name)
config.output_hidden_states = True
# self.bert = BertModel.from_pretrained("bert-base-cased")
self.model = AutoModelWithLMHead.from_pretrained(params.model_name, config=config)
if params.ckpt != "":
logger.info("Reloading model from %s" % params.ckpt)
model_ckpt = torch.load(params.ckpt)
self.model.load_state_dict(model_ckpt)
self.linear = nn.Linear(self.hidden_dim, self.num_tag)
def init_model(model_name: str,
device: torch.device):
"""
Initialize a pre-trained LM
:param model_name: from MODEL_CLASSES
:param device: CUDA / CPU device
:return: the model and tokenizer
"""
logger.info(f'Initializing {model_name}')
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelWithLMHead.from_pretrained(model_name)
model.to(device)
model.eval()
return model, tokenizer
def load_model(self):
if 'longformer' in self.args.model_path:
model = LongformerModel.from_pretrained(self.args.model_path)
for layer in model.encoder.layer:
layer.attention.self.attention_mode = self.args.attention_mode
self.args.attention_window = 512 # layer.attention.self.attention_window
elif self.args.model_path in ['bart.large', 'bart.base']:
model = torch.hub.load('pytorch/fairseq', self.args.model_path)
model.config = model.args
model.config.hidden_size = model.config.decoder_output_dim
elif 'bart' in self.args.model_path and 'base' in self.args.model_path:
config = AutoConfig.from_pretrained(self.args.model_path)
config.encoder_attention_heads = 12
config.decoder_attention_heads = 12
config.attention_dropout = 0.1
if self.args.seq2seq:
model = AutoModelWithLMHead.from_pretrained(self.args.model_path, config=config)
else:
model = AutoModel.from_pretrained(self.args.model_path, config=config)
elif 'bart' in self.args.model_path and 'large' in self.args.model_path:
config = AutoConfig.from_pretrained(self.args.model_path)
config.attention_dropout = 0.1
config.gradient_checkpointing = True
if self.args.seq2seq:
model = AutoModelWithLMHead.from_pretrained(self.args.model_path, config=config)
else:
model = AutoModel.from_pretrained(self.args.model_path, config=config)
else:
model = AutoModel.from_pretrained(self.args.model_path)
print("Loaded model with config:")
print(model.config)
for p in model.parameters():
p.requires_grad_(True)
model.train()
return model
def _parallelize_summarizations(article_filenames: List[str]) -> None:
model = AutoModelWithLMHead.from_pretrained("t5-base", return_dict=True)
model.share_memory()
tokenizer = AutoTokenizer.from_pretrained("t5-base")
"""
# Multiprocessing #1
# Hangs on Ubuntu machine for some reason (tried using torch.multiprocessing, moving outside of main method, using model and tokenizer as params, etc; nothing works)
# For some reason, worked fine in 40-45 seconds on FB Mac
# 102-105 seconds
mp.set_start_method("spawn")
with Pool(NUM_PROCESSES) as p:
article_summaries = p.starmap(
_summarize_news_article,
[
(model, tokenizer, article_filename)
for article_filename in article_filenames
],
)
print(f"Article summaries: {article_summaries}")
p.close()
p.join()
"""
# Multiprocesing #2
# Issue #1: https://stackoverflow.com/questions/50168647/multiprocessing-causes-python-to-crash-and-gives-an-error-may-have-been-in-progr
# 40-45 seconds
# 90-105 seconds (more variance than multiprocessing #1, but first approach should be better since it's better to fix # processes created)
"""
procs = []
for article_file in article_files:
proc = Process(
target=_summarize_news_article, args=(model, tokenizer, article_file,)
)
procs.append(proc)
proc.start()
for proc in procs:
proc.join()
"""
# List Comprehension
# 90-95 seconds
# ~220 seconds
article_summaries = [
_summarize_news_article(model, tokenizer, article_file)
for article_file in article_filenames
]
for article_summary in article_summaries:
print(f"Article summary: {article_summary}")
def main(politician, epochs):
"""
High-level management of model training process.
"""
train_path = f"..\\data\\{politician}\\training_data.txt"
val_path = f"..\\data\\{politician}\\validation_data.txt"
tokenizer = AutoTokenizer.from_pretrained(
"anonymous-german-nlp/german-gpt2")
special_tokens_dict = {
'bos_token': '<BOS>',
'eos_token': '<EOS>',
'pad_token': '<PAD>',
'additional_special_tokens': ['<EOQ>']
}
tokenizer.add_special_tokens(special_tokens_dict)
train_dataset, test_dataset, data_collator = load_dataset(
train_path, val_path, tokenizer)
model = AutoModelWithLMHead.from_pretrained(
"anonymous-german-nlp/german-gpt2")
model.resize_token_embeddings(len(tokenizer))
training_args = TrainingArguments(
output_dir=
f".\\output-models\\gpt2-{politician}-{epochs}", # output directory
overwrite_output_dir=
True, # overwrite the content of the output directory
num_train_epochs=epochs, # number of training epochs
per_device_train_batch_size=32, # batch size for training
per_device_eval_batch_size=64, # batch size for evaluation
eval_steps=400, # Number of update steps between two evaluations.
save_steps=800, # after # steps model is saved
warmup_steps=500, # number of warmup steps for learning rate scheduler
)
trainer = Trainer(
model=model,
args=training_args,
data_collator=data_collator,
train_dataset=train_dataset,
eval_dataset=test_dataset,
prediction_loss_only=True,
)
trainer.train()
trainer.save_model()
def train():
dialogues = pd.read_csv('/content/TlkPersonaChatRus/dialogues.tsv',
sep='\t')
for column in dialogues.columns:
dialogues[column].replace(to_replace=r'<[a-zA-Z0-9_=\/ ]+>',
value=' ',
regex=True,
inplace=True)
dialogues['dialogue'].replace(
to_replace=r'Пользователь [12]:|Привет|Здравствуйте|[!)?,]',
value='',
regex=True,
inplace=True)
dialogues['dialogue'].replace(to_replace=r'\s\s+',
value=' ',
regex=True,
inplace=True)
dialogues = dialogues['dialogue']
dialogues.to_csv('./Datasets/dialogues')
tokenizer = AutoTokenizer.from_pretrained('distilgpt2')
model = AutoModelWithLMHead.from_pretrained('distilgpt2')
tokenizer.pad_token = tokenizer.eos_token
dataset = LineByLineTextDataset(
tokenizer=tokenizer,
file_path=
'/content/drive/MyDrive/semester-practice-3rd/Datasets/dialogues.txt',
block_size=128,
)
data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer,
mlm=False)
training_args = TrainingArguments(
output_dir=
'/content/drive/MyDrive/semester-practice-3rd/Models/distilgpt2',
overwrite_output_dir=True,
num_train_epochs=10,
per_device_train_batch_size=8,
)
trainer = Trainer(model=model,
args=training_args,
data_collator=data_collator,
train_dataset=dataset)
trainer.train()
trainer.save_model('model/gpt2_chat')
def nlp_it (text, lenght):
tokenizer = AutoTokenizer.from_pretrained('t5-base')
model = AutoModelWithLMHead.from_pretrained('t5-base', return_dict=True)
inputs = tokenizer.encode(
"summarize: " + text,
return_tensors='pt',
max_length=512,
truncation=True
)
summary_ids = model.generate(inputs, max_length=lenght, min_length=80, length_penalty=5., num_beams=2)
summary = tokenizer.decode(summary_ids[0])
return summary
def main():
# Example: use pre-trained model from Huggingface
tokenizer = AutoTokenizer.from_pretrained("MoseliMotsoehli/TswanaBert")
model = AutoModelWithLMHead.from_pretrained("MoseliMotsoehli/TswanaBert")
# On own model: make sure you trained a model with Masking (The current Roberta model has no masking)
#tokenizer = AutoTokenizer.from_pretrained("./tswana_models/output")
#model = AutoModel.from_pretrained("./tswana_models/output", output_attentions=True)
unmasker = pipeline('fill-mask', model=model, tokenizer=tokenizer)
text = 'Ka ponyo ya <mask> lefatshe la nona.'
# Ka ponyo ya leitlho lefatshe la nona
#text = 'Ke bereketse kwa <mask> lobaka lo lo leele.'
#'Ke bereketse kwa Kanye lobaka lo lo leele'
print(text, unmasker(text))
def test_lmhead_model_from_pretrained(self):
for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
config = AutoConfig.from_pretrained(model_name)
self.assertIsNotNone(config)
self.assertIsInstance(config, BertConfig)
model = TFAutoModelWithLMHead.from_pretrained(model_name,
from_pt=True)
self.assertIsNotNone(model)
self.assertIsInstance(model, TFBertForMaskedLM)
model = AutoModelWithLMHead.from_pretrained(model_name,
from_tf=True)
self.assertIsNotNone(model)
self.assertIsInstance(model, BertForMaskedLM)
def __init__(self,
model_name="bert-base-cased-finetuned-mrpc",
task="SequenceClassification"):
self.tokenizer = AutoTokenizer.from_pretrained(model_name)
if task == "SC":
self.model = AutoModelForSequenceClassification.from_pretrained(
model_name)
elif task == "QA":
self.model = AutoModelForQuestionAnswering.from_pretrained(
model_name)
elif task == "LM":
self.model = AutoModelWithLMHead.from_pretrained(model_name)
elif task == "TC":
self.model = AutoModelForTokenClassification.from_pretrained(
"dbmdz/bert-large-cased-finetuned-conll03-english")
def __init__(self, pretrained_model_name, config, skip_mlm=False):
super(XLMForMTBPreTraining, self).__init__()
self.config = config
self.skip_mlm = skip_mlm
# Add the entity markers and BLANKS tokens
model = AutoModelWithLMHead.from_pretrained(pretrained_model_name)
self.encoder = model.transformer
self.encoder.resize_token_embeddings(self.config.vocab_size)
if not self.skip_mlm:
self.mlm_head = model.pred_layer
self.mlm_loss_fn = nn.CrossEntropyLoss()
self.re_head = REHead(self.config)
self.mtb_loss_fn = nn.BCEWithLogitsLoss()
def modelWithLMHead(*args, **kwargs):
r"""
# Using torch.hub !
import torch
model = torch.hub.load('huggingface/transformers', 'modelWithLMHead', 'bert-base-uncased') # Download model and configuration from S3 and cache.
model = torch.hub.load('huggingface/transformers', 'modelWithLMHead', './test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')`
model = torch.hub.load('huggingface/transformers', 'modelWithLMHead', 'bert-base-uncased', output_attention=True) # Update configuration during loading
assert model.config.output_attention == True
# Loading from a TF checkpoint file instead of a PyTorch model (slower)
config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json')
model = torch.hub.load('huggingface/transformers', 'modelWithLMHead', './tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config)
"""
return AutoModelWithLMHead.from_pretrained(*args, **kwargs)
def loadModels(splittingModelPath, waypointModelPath):
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
model = AutoModelWithLMHead.from_pretrained("bert-base-uncased")
splittingNet = SplittingNet(model.bert)
splittingNet.load_state_dict(
torch.load(splittingModelPath, map_location=torch.device('cpu')))
splittingNet.eval()
waypointNet = WaypointNet(model.bert)
waypointNet.load_state_dict(
torch.load(waypointModelPath, map_location=torch.device('cpu')))
waypointNet.eval()
return (splittingNet, waypointNet, tokenizer)
def __init__(self, model_scale):
self.device = torch.device(
"cuda" if torch.cuda.is_available() else "cpu")
#self.device = "cpu"
print("Device is " + str(self.device))
#self.tokenizer = GPT2Tokenizer.from_pretrained("distilgpt2")
#self.model = GPT2LMHeadModel.from_pretrained("distilgpt2")
if model_scale == 0:
self.tokenizer = AutoTokenizer.from_pretrained("distilgpt2")
self.model = AutoModelWithLMHead.from_pretrained("distilgpt2")
self.scorer = LMScorer.from_pretrained("distilgpt2",
device=self.device)
elif model_scale == 1:
self.tokenizer = AutoTokenizer.from_pretrained("gpt2")
self.model = AutoModelWithLMHead.from_pretrained("gpt2")
self.scorer = LMScorer.from_pretrained("gpt2", device=self.device)
elif model_scale == 2:
self.tokenizer = AutoTokenizer.from_pretrained("gpt2-medium")
self.model = AutoModelWithLMHead.from_pretrained("gpt2-medium")
self.scorer = LMScorer.from_pretrained("gpt2-medium",
device=self.device)
elif model_scale == 3:
self.tokenizer = AutoTokenizer.from_pretrained("gpt2-large")
self.model = AutoModelWithLMHead.from_pretrained("gpt2-large")
self.scorer = LMScorer.from_pretrained("gpt2-large",
device=self.device)
else:
self.tokenizer = AutoTokenizer.from_pretrained("gpt2-xl")
self.model = AutoModelWithLMHead.from_pretrained("gpt2-xl")
self.scorer = LMScorer.from_pretrained("gpt2-xl",
device=self.device)
#self.model.eval()
self.model.to(self.device)
def __init__(self, transformer_model, random_seed):
random.seed(random_seed)
np.random.seed(random_seed)
torch.manual_seed(random_seed)
self.random_seed = random_seed
self.model = AutoModelWithLMHead.from_pretrained(transformer_model)
self.tokenizer = AutoTokenizer.from_pretrained(transformer_model)
self.terms = []
self.embeddings = torch.FloatTensor([])
self.embeddings_2d = None
self.diffs = []
self.embed = None
self.sim_fn = torch.nn.CosineSimilarity(dim=1)
def load_from_pretrained():
try:
print("*" * 50)
print("Load from checkpoint")
tokenizer = GPT2Tokenizer.from_pretrained(opts.model_name_or_path,
pad_token='<|endoftext|>',
cls_token='<|cls|>',
sep_token='<|sep|>')
model = GPT2LMHeadModel.from_pretrained(opts.model_name_or_path)
try:
with open(os.path.join(opts.output_dir, 'stats.pkl'), 'rb') as f:
stats = pickle.load(f)
except:
print("Can't find training stats...")
stats = None
print("*" * 50)
except Exception as e:
print(e)
try:
# from dialogpt pretrained
print("*" * 50)
print("Load from pretrained")
print("*" * 50)
tokenizer = GPT2Tokenizer.from_pretrained(
tokenizer_path,
pad_token='<|endoftext|>',
cls_token='<|cls|>',
sep_token='<|sep|>')
model = GPT2LMHeadModel.from_pretrained(model_path)
except:
print("*" * 50)
print("Downloading ... ")
print("*" * 50)
# download dialogpt
tokenizer = AutoTokenizer.from_pretrained(
opts.download_name,
pad_token='<|endoftext|>',
cls_token='<|cls|>',
sep_token='<|sep|>')
model = AutoModelWithLMHead.from_pretrained(opts.download_name)
# save to dialogpt
tokenizer.save_pretrained(tokenizer_path)
model.save_pretrained(model_path)
stats = None
tokenizer.add_special_tokens(
{'additional_special_tokens': ['<|start|>', '<|p1|>', '<|p2|>']})
model.resize_token_embeddings(len(tokenizer))
return model.to(device), tokenizer, stats
def _set_language_model_and_tokenizer(self,
pretrained_weights: str) -> None:
self._pretrained_weights = pretrained_weights
try:
self.language_model = BertForMaskedLM.from_pretrained(
self._pretrained_weights, cache_dir=self._lm_dir)
self.tokenizer = BertTokenizer.from_pretrained(
self._pretrained_weights, cache_dir=self._lm_dir)
except OSError:
print('{} requires AutoModelWithLMHead and AutoTokenizer'.format(
pretrained_weights))
self.language_model = AutoModelWithLMHead.from_pretrained(
self._pretrained_weights, cache_dir=self._lm_dir)
self.tokenizer = AutoTokenizer.from_pretrained(
self._pretrained_weights, cache_dir=self._lm_dir)
def _convert_to_transformers_lm(adaptive_model, prediction_head):
# init model
transformers_model = AutoModelWithLMHead.from_config(adaptive_model.language_model.model.config)
# transfer weights for language model + prediction head
setattr(transformers_model, transformers_model.base_model_prefix, adaptive_model.language_model.model)
# Adding decoder bias (required for conversion to transformers)
prediction_head.decoder.bias = prediction_head.bias
ph_state_dict = prediction_head.state_dict()
ph_state_dict["transform.dense.weight"] = ph_state_dict.pop("dense.weight")
ph_state_dict["transform.dense.bias"] = ph_state_dict.pop("dense.bias")
ph_state_dict["transform.LayerNorm.weight"] = ph_state_dict.pop("LayerNorm.weight")
ph_state_dict["transform.LayerNorm.bias"] = ph_state_dict.pop("LayerNorm.bias")
transformers_model.cls.predictions.load_state_dict(ph_state_dict)
return transformers_model
def transformers_svc():
"""Return a Transformers BentoService."""
# When the ExampleBentoService got saved and loaded again in the test, the
# two class attribute below got set to the loaded BentoService class.
# Resetting it here so it does not effect other tests
TransformersGPT2TextGenerator._bento_service_bundle_path = None
TransformersGPT2TextGenerator._bento_service_bundle_version = None
svc = TransformersGPT2TextGenerator()
tokenizer = AutoTokenizer.from_pretrained("gpt2")
model = AutoModelWithLMHead.from_pretrained(
"gpt2", pad_token_id=tokenizer.eos_token_id)
model_artifact = {"model": model, "tokenizer": tokenizer}
svc.pack("gptModel", model_artifact)
return svc
def __init__(self, name='T5 Translator', task='translation_en_to_de', device="cpu"):
super().__init__(name)
#Init name and metadata
self.name = name
self.task = task
# self.device = 1 if torch.cuda.is_available() else -1
self.device = -1 if device.lower() == "cpu" else 1
#Create net
self.tokenizer = AutoTokenizer.from_pretrained("t5-base")
self.model = AutoModelWithLMHead.from_pretrained("t5-base")
self.predictor = pipeline(self.task,
model = self.model,
tokenizer = self.tokenizer,
device = self.device)
def run_chinese_bert_wwm_torch():
import torch
tokenizer = AutoTokenizer.from_pretrained("hfl/chinese-bert-wwm-ext")
model = AutoModelWithLMHead.from_pretrained("hfl/chinese-bert-wwm-ext")
input_ids = torch.tensor(tokenizer.encode("[MASK][MASK][MASK]总书记讲到三个关联“做好[MASK][MASK]防控工作，直接关系[MASK][MASK]生命安全和身体健康，直接关系经济社会大局稳定，也事关我国对外开放”", add_special_tokens=True)).unsqueeze(
0) # Batch size 1
# print(f"input_ids:{input_ids}")
outputs = model(input_ids)
logits = outputs[0]
# print(logits)
# print("-" * 30)
# print(f"outputs:{outputs}")
print("-" * 30)
# print(tf.math.argmax(logits, axis=2)[0, :])
print(tokenizer.decode(torch.argmax(logits, dim=2)[0, :]))
def __init__(self, brown_ic=None, glove_vecs=None, word_vectors=None):
#write any initializing code here
# DEFINE THRESHOLD VALUE
self.dist_threshold = 0.3
# 1. GET EMBEDDING FOR RED WORDS USING GPT2
torch.set_grad_enabled(False)
self.tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
self.model = AutoModelWithLMHead.from_pretrained('gpt2')
#get stop words and what-not
nltk.download('popular', quiet=True)
nltk.download('words', quiet=True)
self.corp_words = set(nltk.corpus.words.words())
return
def main(config: DictConfig) -> None:
print(config)
tokenizer = AutoTokenizer.from_pretrained("imthanhlv/gpt2news")
tokenizer.pad_token = tokenizer.eos_token
tokenizer.padding_side = "right"
gpt2 = AutoModelWithLMHead.from_pretrained("imthanhlv/gpt2news")
gpt2.resize_token_embeddings(len(tokenizer))
gpt2.config.pad_token_id = gpt2.config.eos_token_id
corpus = UITABSAHotel()
# num_labels = corpus.num_labels
num_labels = corpus.num_aspect_labels
model = GPT2TextClassification(gpt2, num_labels)
datamodule = MultiLabelClassificationDatamodule(corpus=corpus, tokenizer=tokenizer, **config.data)
logger = WandbLogger(**config.logger)
trainer = pl.Trainer(logger=logger, **config.trainer)
trainer.fit(model, datamodule=datamodule)
def load_model(model_path, tokenizer_path=None, no_cuda=False, quantize=False):
'''
Loads a pretrained language model and tokenzier.
:param model_path:
The name of standard pretrained model or a path
to a checkpoint for weights initialization.
:param tokenizer_path:
Optional pretrained tokenizer name or path if not
the same as the model checkpoint path. If both None,
a new tokenizer will be initialized.
:param no_cuda:
Disable CUDA devices even when they are available. Defaults to False.
:param quantize:
Indicates whether to load quantize the model.
:returns:
A :class:`transformers.PreTrainedModel` and a
:class:`transformers.PreTrainedTokenizer`.
'''
# Setup device
device = torch.device(
'cuda' if torch.cuda.is_available() and not no_cuda else 'cpu')
if quantize and device != 'cpu':
raise RuntimeError('Model quantization only available on CPU devices.')
if tokenizer_path:
tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)
elif model_path:
tokenizer = AutoTokenizer.from_pretrained(model_path)
else:
raise ValueError(
'Instantiating a new tokenizer from scratch is not support; however, it can be done from another script.'
'Use the tokenizer_path argument to provide it with the location of the script to load the tokenizer.'
)
model = AutoModelWithLMHead.from_pretrained(model_path)
if quantize:
model = torch.quantization.quantize_dynamic(model, {
torch.nn.Linear, torch.nn.Embedding,
transformers.modeling_utils.Conv1D
},
dtype=torch.qint8)
return model.to(device), tokenizer
def get_transformer(LM: bool):
if LM:
if model_name == 'distilbert-base-cased':
model = TFDistilBertForMaskedLM.from_pretrained(
'distilbert-base-cased')
elif model_name == 'huggingface/CodeBERTa-small-v1':
model = AutoModelWithLMHead.from_pretrained(
'huggingface/CodeBERTa-small-v1')
model = pt_to_tf(model, TFRobertaForMaskedLM)
else:
if model_name == 'distilbert-base-cased':
model = TFDistilBertModel.from_pretrained(
'distilbert-base-cased', )
elif model_name == 'huggingface/CodeBERTa-small-v1':
model = AutoModel.from_pretrained('huggingface/CodeBERTa-small-v1')
model = pt_to_tf(model, TFRobertaModel)
return model
