def __init__(
self,
config: Optional[PretrainedConfig] = None,
encoder: Optional[PreTrainedModel] = None,
decoder: Optional[PreTrainedModel] = None,
):
assert config is not None or (
encoder is not None and decoder is not None
), "Either a configuration or an Encoder and a decoder has to be provided"
if config is None:
config = EncoderDecoderConfig.from_encoder_decoder_configs(encoder.config, decoder.config)
else:
assert isinstance(config, self.config_class), "config: {} has to be of type {}".format(
config, self.config_class
)
# initialize with config
super().__init__(config)
if encoder is None:
from transformers import AutoModel
encoder = AutoModel.from_config(config.encoder)
if decoder is None:
from transformers import AutoModelWithLMHead
decoder = AutoModelWithLMHead.from_config(config.decoder)
self.encoder = encoder
self.decoder = decoder
assert (
self.encoder.get_output_embeddings() is None
), "The encoder {} should not have a LM Head. Please use a model without LM Head"
def __init__(self, evaluate: bool = False):
self.model_args = ModelArguments(model_name_or_path=None, model_type='bert', tokenizer_name='models/danbert-small/vocab.json', config_name="models/danbert-small/config.json")
self.data_args = DataTrainingArguments(train_data_file="handler/datadir/da-train.txt", eval_data_file="handler/datadir/da-eval.txt", labels="handler/datadir/labels.txt", mlm=True, line_by_line=True)
self.training_args = TrainingArguments(output_dir="models/danbert-small", num_train_epochs=3, per_gpu_eval_batch_size=8, save_steps=750, seed=42, learning_rate=1e-4, save_total_limit=2)
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO if self.training_args.local_rank in [-1, 0] else logging.WARN,
)
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
self.training_args.local_rank,
self.training_args.device,
self.training_args.n_gpu,
bool(self.training_args.local_rank != -1),
self.training_args.fp16,
)
logger.info("Training/evaluation parameters %s", self.training_args)
set_seed(self.training_args.seed)
config = CONFIG_MAPPING[self.model_args.model_type]()
logger.warning("You are instantiating a new config instance from scratch.")
tokenizer = AutoTokenizer.from_pretrained('models/danbert-small')
model = AutoModelWithLMHead.from_config(config)
# model.resize_token_embeddings(len(tokenizer))
if self.data_args.block_size <= 0:
self.data_args.block_size = 512
# Our input block size will be the max possible for the model
else:
self.data_args.block_size = min(self.data_args.block_size, 512)
train_dataset = (
self.get_dataset(self.data_args, tokenizer=tokenizer, local_rank=self.training_args.local_rank, evaluate=True)
)
eval_dataset = (
self.get_dataset(self.data_args, tokenizer=tokenizer, local_rank=self.training_args.local_rank, evaluate=True)
)
data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=self.data_args.mlm, mlm_probability=self.data_args.mlm_probability)
trainer = Trainer(
model=model,
args=self.training_args,
data_collator=data_collator,
train_dataset=train_dataset,
# eval_dataset=eval_dataset,
prediction_loss_only=True,
)
trainer.train(model_path=None)
trainer.save_model()
def load_model(model_args, config):
if model_args.model_name_or_path:
model = AutoModelWithLMHead.from_pretrained(
model_args.model_name_or_path,
from_tf=bool(".ckpt" in model_args.model_name_or_path),
config=config,
cache_dir=model_args.cache_dir,
)
else:
logger.info("Training new model from scratch")
model = AutoModelWithLMHead.from_config(config)
return model
def __init__(self, hparams):
super().__init__(hparams)
self.tokenizer = AutoTokenizer.from_pretrained(
"huggingface/CodeBERTa-small-v1", resume_download=True)
# FIXME: using mask lm head in pretraining, but using automodel in finetuning.
# self.lm_model = AutoModel.from_pretrained(
# "huggingface/CodeBERTa-small-v1", resume_download=True, config=hparams["roberta_config"])
self.model = AutoModelWithLMHead.from_config(
AutoConfig.from_pretrained("huggingface/CodeBERTa-small-v1",
resume_download=True,
config=hparams["roberta_config"]))
def _convert_to_transformers_lm(adaptive_model, prediction_head):
# init model
transformers_model = AutoModelWithLMHead.from_config(adaptive_model.language_model.model.config)
# transfer weights for language model + prediction head
setattr(transformers_model, transformers_model.base_model_prefix, adaptive_model.language_model.model)
# Adding decoder bias (required for conversion to transformers)
prediction_head.decoder.bias = prediction_head.bias
ph_state_dict = prediction_head.state_dict()
ph_state_dict["transform.dense.weight"] = ph_state_dict.pop("dense.weight")
ph_state_dict["transform.dense.bias"] = ph_state_dict.pop("dense.bias")
ph_state_dict["transform.LayerNorm.weight"] = ph_state_dict.pop("LayerNorm.weight")
ph_state_dict["transform.LayerNorm.bias"] = ph_state_dict.pop("LayerNorm.bias")
transformers_model.cls.predictions.load_state_dict(ph_state_dict)
return transformers_model
def get_model(args, device, vocabulary=None, config=None):
if any(mn in args.model_name for mn in args.transformers_models):
if config is None:
config = get_config(args, vocabulary)
config.n_ctx = args.seq_size
model = AutoModelWithLMHead.from_config(config)
model = model.to(device)
elif args.model_name in 'sinkhorn':
model = SinkhornTransformerLMGenerator(
num_tokens=args.vocab_size,
dim=1024,
heads=8,
depth=12,
bucket_size=20, # size of the buckets
causal=False, # auto-regressive or not
n_sortcut=2, # use sortcut to reduce memory complexity to linear
ff_chunks=10, # feedforward chunking, from Reformer paper
reversible=True, # make network reversible, from Reformer paper
ff_dropout=0.1, # feedforward dropout
attn_dropout=0.1, # post attention dropout
attn_layer_dropout=0.1, # post attention layer dropout
layer_dropout=0.1, # add layer dropout, from 'Reducing Transformer Depth on Demand' paper
weight_tie=True, # tie layer parameters, from Albert paper
emb_dim=128, # embedding factorization, from Albert paper
ff_glu=True, # use GLU in feedforward, from paper 'GLU Variants Improve Transformer'
n_local_attn_heads=2,
# replace N heads with local attention, suggested to work well from Routing Transformer paper
max_seq_len=args.seq_size
).to(device)
elif args.model_name in 'gru':
model = BaseGru(args.vocab_size, 1024, args.batch_size)
elif args.model_name in 'seq2seq':
model = Seq2SeqRNN_attn(256, 256, 1024, 100, 3, args.vocab_size+1, args.vocab_size+2, args.vocab_size)
return model
def main():
# See all possible arguments in src/transformers/training_args.py
# or by passing the --help flag to this script.
# We now keep distinct sets of args, for a cleaner separation of concerns.
parser = HfArgumentParser(
(ModelArguments, DataTrainingArguments, TrainingArguments))
model_args, data_args, training_args = parser.parse_args_into_dataclasses()
if data_args.eval_data_file is None and training_args.do_eval:
raise ValueError(
"Cannot do evaluation without an evaluation data file. Either supply a file to --eval_data_file "
"or remove the --do_eval argument.")
if (os.path.exists(training_args.output_dir)
and os.listdir(training_args.output_dir) and training_args.do_train
and not training_args.overwrite_output_dir):
raise ValueError(
f"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
)
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO
if training_args.local_rank in [-1, 0] else logging.WARN,
)
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
training_args.local_rank,
training_args.device,
training_args.n_gpu,
bool(training_args.local_rank != -1),
training_args.fp16,
)
logger.info("Training/evaluation parameters %s", training_args)
# Set seed
set_seed(training_args.seed)
# Load pretrained model and tokenizer
#
# Distributed training:
# The .from_pretrained methods guarantee that only one local process can concurrently
# download model & vocab.
if model_args.config_name:
config = AutoConfig.from_pretrained(model_args.config_name,
cache_dir=model_args.cache_dir)
elif model_args.model_name_or_path:
config = AutoConfig.from_pretrained(model_args.model_name_or_path,
cache_dir=model_args.cache_dir)
else:
config = CONFIG_MAPPING[model_args.model_type]()
logger.warning(
"You are instantiating a new config instance from scratch.")
if model_args.tokenizer_name:
tokenizer = AutoTokenizer.from_pretrained(
model_args.tokenizer_name, cache_dir=model_args.cache_dir)
elif model_args.model_name_or_path:
tokenizer = AutoTokenizer.from_pretrained(
model_args.model_name_or_path, cache_dir=model_args.cache_dir)
else:
raise ValueError(
"You are instantiating a new tokenizer from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --tokenizer_name")
if model_args.model_name_or_path:
model = AutoModelWithLMHead.from_pretrained(
model_args.model_name_or_path,
from_tf=bool(".ckpt" in model_args.model_name_or_path),
config=config,
cache_dir=model_args.cache_dir,
)
else:
logger.info("Training new model from scratch")
model = AutoModelWithLMHead.from_config(config)
if model_args.gen_model_name_or_path:
gen_config = AutoConfig.from_pretrained(
model_args.gen_model_name_or_path,
num_labels=2,
id2label={
0: "0",
1: "1"
},
label2id={
"0": 0,
"1": 1
},
cache_dir=model_args.cache_dir,
)
gen_model = AutoModelForTokenClassification.from_pretrained(
model_args.gen_model_name_or_path,
from_tf=bool(".ckpt" in model_args.gen_model_name_or_path),
config=gen_config,
cache_dir=model_args.cache_dir,
)
gen_model.resize_token_embeddings(len(tokenizer))
mask_generator = MaskGenerator(gen_model, training_args)
if model_args.cls_model_name_or_path:
cls_config = AutoConfig.from_pretrained(
model_args.cls_model_name_or_path,
num_labels=2,
finetuning_task="cola",
cache_dir=model_args.cache_dir,
)
cls_model = AutoModelForSequenceClassification.from_pretrained(
model_args.cls_model_name_or_path,
from_tf=bool(".ckpt" in model_args.cls_model_name_or_path),
config=cls_config,
cache_dir=model_args.cache_dir,
)
cls_model.resize_token_embeddings(len(tokenizer))
mask_selector = MaskSelector(cls_model, training_args)
model.resize_token_embeddings(len(tokenizer))
if config.model_type in ["bert", "roberta", "distilbert", "camembert"
] and not data_args.mlm:
raise ValueError(
"BERT and RoBERTa-like models do not have LM heads but masked LM heads. They must be run using the --mlm "
"flag (masked language modeling).")
if data_args.block_size <= 0:
data_args.block_size = tokenizer.max_len
# Our input block size will be the max possible for the model
else:
data_args.block_size = min(data_args.block_size, tokenizer.max_len)
# Get datasets
train_dataset = get_dataset(
data_args, tokenizer=tokenizer,
model_args=model_args) if training_args.do_train else None
eval_dataset = get_dataset(
data_args, model_args=None, tokenizer=tokenizer,
evaluate=True) if training_args.do_eval else None
if model_args.cls_model_name_or_path:
data_collator = DataCollatorForCheckMaskGen(
tokenizer=tokenizer,
mlm=data_args.mlm,
mlm_probability=data_args.mlm_probability,
generator=mask_generator,
selector=mask_selector)
train_sampler = (RandomSampler(train_dataset)
if training_args.local_rank == -1 else
DistributedSampler(train_dataset))
train_dataloader = DataLoader(
train_dataset,
batch_size=128,
sampler=train_sampler,
collate_fn=data_collator.collate_batch,
drop_last=False,
)
all_scores = []
epoch_iterator = tqdm(train_dataloader)
for score in epoch_iterator:
# print(score)
all_scores.extend(score)
print("avg tgt score:")
print(sum(all_scores) / len(all_scores))
exit()
data_collator = DataCollatorForMaskGen(
tokenizer=tokenizer,
mlm=data_args.mlm,
mlm_probability=data_args.mlm_probability,
generator=mask_generator)
# Initialize our Trainer
trainer = Trainer(
model=model,
args=training_args,
data_collator=data_collator,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
prediction_loss_only=True,
)
# Training
if training_args.do_train:
model_path = (model_args.model_name_or_path
if model_args.model_name_or_path is not None
and os.path.isdir(model_args.model_name_or_path) else
None)
trainer.train(model_path=model_path)
trainer.save_model()
# For convenience, we also re-save the tokenizer to the same directory,
# so that you can share your model easily on huggingface.co/models =)
if trainer.is_world_master():
tokenizer.save_pretrained(training_args.output_dir)
# Evaluation
results = {}
if training_args.do_eval:
logger.info("*** Evaluate ***")
eval_output = trainer.evaluate()
perplexity = math.exp(eval_output["eval_loss"])
result = {"perplexity": perplexity}
output_eval_file = os.path.join(training_args.output_dir,
"eval_results_lm.txt")
if trainer.is_world_master():
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
results.update(result)
return results
def main():
parser = argparse.ArgumentParser()
# Required parameters
parser.add_argument("--train_data_file",
default=None,
type=str,
required=True,
help="The input training data file (a text file).")
parser.add_argument(
"--output_dir",
type=str,
required=True,
help=
"The output directory where the model predictions and checkpoints will be written.",
)
parser.add_argument(
"--model_type",
type=str,
required=True,
help="The model architecture to be trained or fine-tuned.",
)
# Other parameters
parser.add_argument(
"--eval_data_file",
default=None,
type=str,
help=
"An optional input evaluation data file to evaluate the perplexity on (a text file).",
)
parser.add_argument(
"--line_by_line",
action="store_true",
help=
"Whether distinct lines of text in the dataset are to be handled as distinct sequences.",
)
parser.add_argument(
"--should_continue",
action="store_true",
help="Whether to continue from latest checkpoint in output_dir")
parser.add_argument(
"--model_name_or_path",
default=None,
type=str,
help=
"The model checkpoint for weights initialization. Leave None if you want to train a model from scratch.",
)
parser.add_argument(
"--mlm",
action="store_true",
help=
"Train with masked-language modeling loss instead of language modeling."
)
parser.add_argument(
"--mlm_probability",
type=float,
default=0.15,
help="Ratio of tokens to mask for masked language modeling loss")
parser.add_argument(
"--config_name",
default=None,
type=str,
help=
"Optional pretrained config name or path if not the same as model_name_or_path. If both are None, "
"initialize a new config.",
)
parser.add_argument(
"--tokenizer_name",
default=None,
type=str,
help=
"Optional pretrained tokenizer name or path if not the same as model_name_or_path. If both are None, "
"initialize a new tokenizer.",
)
parser.add_argument(
"--cache_dir",
default=None,
type=str,
help=
"Optional directory to store the pre-trained models downloaded from s3 (instead of the default one)",
)
parser.add_argument(
"--block_size",
default=-1,
type=int,
help="Optional input sequence length after tokenization."
"The training dataset will be truncated in block of this size for training."
"Default to the model max input length for single sentence inputs (take into account special tokens).",
)
parser.add_argument("--do_train",
action="store_true",
help="Whether to run training.")
parser.add_argument("--do_eval",
action="store_true",
help="Whether to run eval on the dev set.")
parser.add_argument(
"--evaluate_during_training",
action="store_true",
help="Run evaluation during training at each logging step.")
parser.add_argument("--per_gpu_train_batch_size",
default=4,
type=int,
help="Batch size per GPU/CPU for training.")
parser.add_argument("--per_gpu_eval_batch_size",
default=4,
type=int,
help="Batch size per GPU/CPU for evaluation.")
parser.add_argument(
"--gradient_accumulation_steps",
type=int,
default=1,
help=
"Number of updates steps to accumulate before performing a backward/update pass.",
)
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--weight_decay",
default=0.01,
type=float,
help="Weight decay if we apply some.")
parser.add_argument("--adam_epsilon",
default=1e-8,
type=float,
help="Epsilon for Adam optimizer.")
parser.add_argument("--max_grad_norm",
default=1.0,
type=float,
help="Max gradient norm.")
parser.add_argument("--num_train_epochs",
default=1.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--max_steps",
default=-1,
type=int,
help=
"If > 0: set total number of training steps to perform. Override num_train_epochs.",
)
parser.add_argument("--warmup_steps",
default=0,
type=int,
help="Linear warmup over warmup_steps.")
parser.add_argument("--logging_steps",
type=int,
default=500,
help="Log every X updates steps.")
parser.add_argument("--save_steps",
type=int,
default=500,
help="Save checkpoint every X updates steps.")
parser.add_argument(
"--save_total_limit",
type=int,
default=None,
help=
"Limit the total amount of checkpoints, delete the older checkpoints in the output_dir, does not delete "
"by default",
)
parser.add_argument(
"--eval_all_checkpoints",
action="store_true",
help=
"Evaluate all checkpoints starting with the same prefix as model_name_or_path ending and ending with "
"step number",
)
parser.add_argument("--no_cuda",
action="store_true",
help="Avoid using CUDA when available")
parser.add_argument("--overwrite_output_dir",
action="store_true",
help="Overwrite the content of the output directory")
parser.add_argument(
"--overwrite_cache",
action="store_true",
help="Overwrite the cached training and evaluation sets")
parser.add_argument("--seed",
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
"--fp16",
action="store_true",
help=
"Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit",
)
parser.add_argument(
"--fp16_opt_level",
type=str,
default="O1",
help=
"For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
"See details at https://nvidia.github.io/apex/amp.html",
)
parser.add_argument("--local_rank",
type=int,
default=-1,
help="For distributed training: local_rank")
parser.add_argument("--server_ip",
type=str,
default="",
help="For distant debugging.")
parser.add_argument("--server_port",
type=str,
default="",
help="For distant debugging.")
args = parser.parse_args()
if args.model_type in ["bert", "roberta", "distilbert", "camembert"
] and not args.mlm:
raise ValueError(
"BERT and RoBERTa-like models do not have LM heads but masked LM heads. They must be run using the --mlm "
"flag (masked language modeling).")
if args.eval_data_file is None and args.do_eval:
raise ValueError(
"Cannot do evaluation without an evaluation data file. Either supply a file to --eval_data_file "
"or remove the --do_eval argument.")
if args.should_continue:
sorted_checkpoints = _sorted_checkpoints(args)
if len(sorted_checkpoints) == 0:
raise ValueError(
"Used --should_continue but no checkpoint was found in --output_dir."
)
else:
args.model_name_or_path = sorted_checkpoints[-1]
if (os.path.exists(args.output_dir) and os.listdir(args.output_dir)
and args.do_train and not args.overwrite_output_dir
and not args.should_continue):
raise ValueError(
"Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome."
.format(args.output_dir))
# Setup distant debugging if needed
if args.server_ip and args.server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
# print("Waiting for debugger attach")
ptvsd.enable_attach(address=(args.server_ip, args.server_port),
redirect_output=True)
ptvsd.wait_for_attach()
# Setup CUDA, GPU & distributed training
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
args.n_gpu = 0 if args.no_cuda else torch.cuda.device_count()
else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
torch.distributed.init_process_group(backend="nccl")
args.n_gpu = 1
args.device = device
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN,
)
# logger.warning(
# "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
# args.local_rank,
# device,
# args.n_gpu,
# bool(args.local_rank != -1),
# args.fp16,
# )
# Set seed
set_seed(args)
# Load pretrained model and tokenizer
if args.local_rank not in [-1, 0]:
torch.distributed.barrier()
# Barrier to make sure only the first process in distributed training download model & vocab
if args.config_name:
config = AutoConfig.from_pretrained(args.config_name,
cache_dir=args.cache_dir)
elif args.model_name_or_path:
config = AutoConfig.from_pretrained(args.model_name_or_path,
cache_dir=args.cache_dir)
else:
# When we release a pip version exposing CONFIG_MAPPING,
# we can do `config = CONFIG_MAPPING[args.model_type]()`.
raise ValueError(
"You are instantiating a new config instance from scratch. This is not supported, but you can do it from "
"another script, save it, "
"and load it from here, using --config_name")
if args.tokenizer_name:
tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_name,
cache_dir=args.cache_dir)
elif args.model_name_or_path:
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path,
cache_dir=args.cache_dir)
else:
raise ValueError(
"You are instantiating a new tokenizer from scratch. This is not supported, but you can do it from "
"another script, save it, "
"and load it from here, using --tokenizer_name")
if args.block_size <= 0:
args.block_size = tokenizer.max_len
# Our input block size will be the max possible for the model
else:
args.block_size = min(args.block_size, tokenizer.max_len)
if args.model_name_or_path:
model = AutoModelWithLMHead.from_pretrained(
args.model_name_or_path,
from_tf=bool(".ckpt" in args.model_name_or_path),
config=config,
cache_dir=args.cache_dir,
)
else:
# logger.info("Training new model from scratch")
model = AutoModelWithLMHead.from_config(config)
model.to(args.device)
if args.local_rank == 0:
torch.distributed.barrier()
# End of barrier to make sure only the first process in distributed training download model & vocab
# logger.info("Training/evaluation parameters %s", args)
# Training
if args.do_train:
if args.local_rank not in [-1, 0]:
torch.distributed.barrier()
# Barrier to make sure only the first process in distributed
# training process the dataset, and the others will use the cache
train_dataset = load_and_cache_examples(args,
tokenizer,
evaluate=False)
if args.local_rank == 0:
torch.distributed.barrier()
global_step, tr_loss = train(args, train_dataset, model, tokenizer)
# logger.info(" global_step = %s, average loss = %s", global_step, tr_loss)
# Saving best-practices: if you use save_pretrained for the model and tokenizer,
# you can reload them using from_pretrained()
if args.do_train and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
# Create output directory if needed
if args.local_rank in [-1, 0]:
os.makedirs(args.output_dir, exist_ok=True)
# logger.info("Saving model checkpoint to %s", args.output_dir)
# Save a trained model, configuration and tokenizer using `save_pretrained()`.
# They can then be reloaded using `from_pretrained()`
model_to_save = (model.module if hasattr(model, "module") else model
) # Take care of distributed/parallel training
model_to_save.save_pretrained(args.output_dir)
tokenizer.save_pretrained(args.output_dir)
# Good practice: save your training arguments together with the trained model
torch.save(args, os.path.join(args.output_dir, "training_args.bin"))
# Load a trained model and vocabulary that you have fine-tuned
model = AutoModelWithLMHead.from_pretrained(args.output_dir)
tokenizer = AutoTokenizer.from_pretrained(args.output_dir)
model.to(args.device)
# Evaluation
results = {}
if args.do_eval and args.local_rank in [-1, 0]:
checkpoints = [args.output_dir]
if args.eval_all_checkpoints:
checkpoints = list(
os.path.dirname(c) for c in sorted(
glob.glob(args.output_dir + "/**/" + WEIGHTS_NAME,
recursive=True)))
# logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN) # Reduce logging
# logger.info("Evaluate the following checkpoints: %s", checkpoints)
for checkpoint in checkpoints:
global_step = checkpoint.split(
"-")[-1] if len(checkpoints) > 1 else ""
prefix = checkpoint.split(
"/")[-1] if checkpoint.find("checkpoint") != -1 else ""
model = AutoModelWithLMHead.from_pretrained(checkpoint)
model.to(args.device)
result = evaluate(args, model, tokenizer, prefix=prefix)
result = dict(
(k + "_{}".format(global_step), v) for k, v in result.items())
results.update(result)
return results
def main():
parser = argparse.ArgumentParser()
# Required parameters
parser.add_argument(
"--output_dir",
type=str,
required=True,
help=
"The output directory where the model predictions and checkpoints will be written.",
)
parser.add_argument(
"--data_dir",
type=str,
required=True,
help="The path to the input data directory.",
)
parser.add_argument(
"--model_type",
type=str,
required=True,
help="The model architecture to be trained or fine-tuned.",
)
# Other parameters
parser.add_argument(
"--eval_data_file",
default=None,
type=str,
help=
"An optional input evaluation data file to evaluate the perplexity on (a text file).",
)
parser.add_argument(
"--model_name_or_path",
default=None,
type=str,
help=
"The model checkpoint for weights initialization. Leave None if you want to train a model from scratch.",
)
parser.add_argument(
"--mlm",
action="store_true",
help=
"Train with masked-language modeling loss instead of language modeling."
)
parser.add_argument(
"--mlm_probability",
type=float,
default=0.15,
help="Ratio of tokens to mask for masked language modeling loss")
parser.add_argument(
"--config_name",
default=None,
type=str,
help=
"Optional pretrained config name or path if not the same as model_name_or_path. If both are None, initialize a new config.",
)
parser.add_argument(
"--tokenizer_name",
default=None,
type=str,
help=
"Optional pretrained tokenizer name or path if not the same as model_name_or_path. If both are None, initialize a new tokenizer.",
)
parser.add_argument(
"--cache_dir",
default=None,
type=str,
help=
"Optional directory to store the pre-trained models downloaded from s3 (instead of the default one)",
)
parser.add_argument(
"--block_size",
default=-1,
type=int,
help="Optional input sequence length after tokenization."
"The training dataset will be truncated in block of this size for training."
"Default to the model max input length for single sentence inputs (take into account special tokens).",
)
parser.add_argument("--per_gpu_eval_batch_size",
default=4,
type=int,
help="Batch size per GPU/CPU for evaluation.")
parser.add_argument(
"--gradient_accumulation_steps",
type=int,
default=1,
help=
"Number of updates steps to accumulate before performing a backward/update pass.",
)
parser.add_argument("--local_rank",
type=int,
default=-1,
help="For distributed training: local_rank")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--weight_decay",
default=0.0,
type=float,
help="Weight decay if we apply some.")
parser.add_argument("--adam_epsilon",
default=1e-8,
type=float,
help="Epsilon for Adam optimizer.")
parser.add_argument("--max_grad_norm",
default=1.0,
type=float,
help="Max gradient norm.")
parser.add_argument("--num_train_epochs",
default=1.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--max_steps",
default=-1,
type=int,
help=
"If > 0: set total number of training steps to perform. Override num_train_epochs.",
)
parser.add_argument("--warmup_steps",
default=0,
type=int,
help="Linear warmup over warmup_steps.")
parser.add_argument("--logging_steps",
type=int,
default=500,
help="Log every X updates steps.")
parser.add_argument("--save_steps",
type=int,
default=500,
help="Save checkpoint every X updates steps.")
parser.add_argument(
"--save_total_limit",
type=int,
default=None,
help=
"Limit the total amount of checkpoints, delete the older checkpoints in the output_dir, does not delete by default",
)
parser.add_argument(
"--eval_all_checkpoints",
action="store_true",
help=
"Evaluate all checkpoints starting with the same prefix as model_name_or_path ending and ending with step number",
)
parser.add_argument("--no_cuda",
action="store_true",
help="Avoid using CUDA when available")
parser.add_argument("--overwrite_output_dir",
action="store_true",
help="Overwrite the content of the output directory")
parser.add_argument(
"--overwrite_cache",
action="store_true",
help="Overwrite the cached training and evaluation sets")
parser.add_argument("--seed",
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
"--fp16",
action="store_true",
help=
"Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit",
)
parser.add_argument(
"--fp16_opt_level",
type=str,
default="O1",
help=
"For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
"See details at https://nvidia.github.io/apex/amp.html",
)
parser.add_argument(
"--temperature",
type=float,
default=1.0,
help=
"temperature of 1.0 has no effect, lower tend toward greedy sampling",
)
parser.add_argument("--num_return_sequences",
type=int,
default=1,
help="The number of samples to generate.")
parser.add_argument(
"--repetition_penalty",
type=float,
default=1.0,
help="primarily useful for CTRL model; in that case, use 1.2")
parser.add_argument("--length", type=int, default=20)
parser.add_argument("--k", type=int, default=0)
parser.add_argument("--p", type=float, default=0.9)
parser.add_argument("--stop_token",
type=str,
default=None,
help="Token at which text generation is stopped")
parser.add_argument("--sep_token",
type=str,
default="<sep>",
help="Token to separate journal and reflection")
args = parser.parse_args()
if args.model_type in ["bert", "roberta", "distilbert", "camembert"
] and not args.mlm:
raise ValueError(
"BERT and RoBERTa-like models do not have LM heads but masked LM heads. They must be run using the --mlm "
"flag (masked language modeling).")
if (os.path.exists(args.output_dir) and os.listdir(args.output_dir)
and args.do_train and not args.overwrite_output_dir
and not args.should_continue):
raise ValueError(
"Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome."
.format(args.output_dir))
# Setup CUDA, GPU & distributed training
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
args.n_gpu = 0 if args.no_cuda else torch.cuda.device_count()
else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
torch.distributed.init_process_group(backend="nccl")
args.n_gpu = 1
args.device = device
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN,
)
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
args.local_rank,
device,
args.n_gpu,
bool(args.local_rank != -1),
args.fp16,
)
# Set seed
set_seed(args)
# Load pretrained model and tokenizer
if args.local_rank not in [-1, 0]:
torch.distributed.barrier(
) # Barrier to make sure only the first process in distributed training download model & vocab
if args.config_name:
config = AutoConfig.from_pretrained(args.config_name,
cache_dir=args.cache_dir)
elif args.model_name_or_path:
config = AutoConfig.from_pretrained(args.model_name_or_path,
cache_dir=args.cache_dir)
else:
# When we release a pip version exposing CONFIG_MAPPING,
# we can do `config = CONFIG_MAPPING[args.model_type]()`.
raise ValueError(
"You are instantiating a new config instance from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --config_name")
if args.tokenizer_name:
tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_name,
cache_dir=args.cache_dir)
elif args.model_name_or_path:
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path,
cache_dir=args.cache_dir)
else:
raise ValueError(
"You are instantiating a new tokenizer from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --tokenizer_name")
if args.block_size <= 0:
args.block_size = tokenizer.max_len
# Our input block size will be the max possible for the model
else:
args.block_size = min(args.block_size, tokenizer.max_len)
if args.model_name_or_path:
model = AutoModelWithLMHead.from_pretrained(
args.model_name_or_path,
from_tf=bool(".ckpt" in args.model_name_or_path),
config=config,
cache_dir=args.cache_dir,
)
else:
logger.info("Training new model from scratch")
model = AutoModelWithLMHead.from_config(config)
model.to(args.device)
if args.local_rank == 0:
torch.distributed.barrier(
) # End of barrier to make sure only the first process in distributed training download model & vocab
logger.info("Training/evaluation parameters %s", args)
# Training
if args.local_rank not in [-1, 0]:
torch.distributed.barrier(
) # Barrier to make sure only the first process in distributed training process the dataset, and the others will use the cache
non_multi_model = model
if args.n_gpu > 1:
model = torch.nn.DataParallel(non_multi_model)
if args.local_rank == 0:
torch.distributed.barrier()
file_path = Path(args.data_dir, args.eval_data_file)
out_file_path = Path(args.data_dir, "output_" + args.eval_data_file)
id_to_json_map = {}
with open(file_path, encoding="utf-8") as f:
lines = []
i = 0
eval_loss = 0.0
nb_eval_steps = 0
for line in tqdm(f, desc="Evaluating"):
out_json = {}
line = json.loads(line)
example_id = line.get("example_id")
question_text = line.get("question_text")
prompt_text = question_text + " " + args.sep_token + " "
encoded_prompt = tokenizer.encode(prompt_text,
add_special_tokens=False,
return_tensors="pt")
encoded_prompt = encoded_prompt.to(args.device)
output_sequences = non_multi_model.generate(
input_ids=encoded_prompt,
max_length=args.length + len(encoded_prompt[0]),
temperature=args.temperature,
top_k=args.k,
top_p=args.p,
repetition_penalty=args.repetition_penalty,
do_sample=True,
num_return_sequences=args.num_return_sequences,
)
if len(output_sequences.shape) > 2:
output_sequences.squeeze_()
generated_sequences = []
for generated_sequence_idx, generated_sequence in enumerate(
output_sequences):
# print("=== GENERATED SEQUENCE {} ===".format(generated_sequence_idx + 1))
# generated_sequence = output_sequences[0]
generated_sequence = generated_sequence.tolist()
# Decode text
text = tokenizer.decode(generated_sequence,
clean_up_tokenization_spaces=True)
# Remove all text after the stop token
if args.stop_token:
text = text[:text.find(args.stop_token)]
# Add the prompt at the beginning of the sequence. Remove the excess text that was used for pre-processing
total_sequence = (prompt_text + text[len(
tokenizer.decode(encoded_prompt[0],
clean_up_tokenization_spaces=True)):])
# print(total_sequence)
out_json["journaling_input"], out_json[
"reflection_output"] = total_sequence.split(
args.sep_token)[:2]
sample_dataset = GenerateTextDataset(tokenizer, total_sequence,
args.block_size)
def collate(examples: List[torch.Tensor]):
if tokenizer._pad_token is None:
return pad_sequence(examples, batch_first=True)
return pad_sequence(examples,
batch_first=True,
padding_value=tokenizer.pad_token_id)
eval_sampler = SequentialSampler(sample_dataset)
eval_dataloader = DataLoader(sample_dataset,
sampler=eval_sampler,
batch_size=1,
collate_fn=collate)
model_lm = model
if args.n_gpu > 1:
model_lm = torch.nn.DataParallel(model_lm)
model_lm.eval()
for batch in eval_dataloader:
inputs, labels = mask_tokens(
batch, tokenizer, args) if args.mlm else (batch, batch)
inputs = inputs.to(args.device)
labels = labels.to(args.device)
with torch.no_grad():
outputs = model_lm(
inputs,
masked_lm_labels=labels) if args.mlm else model_lm(
inputs, labels=labels)
lm_loss = outputs[0]
example_loss = lm_loss.mean().item()
eval_loss += example_loss
nb_eval_steps += 1
perplexity = torch.exp(torch.tensor(example_loss)).item()
# print(perplexity)
out_json["perplexity"] = perplexity
example_id += "-" + str(generated_sequence_idx)
id_to_json_map[example_id] = json.dumps(out_json,
ensure_ascii=False)
# result = {"perplexity": perplexity}
eval_loss = eval_loss / nb_eval_steps
total_perplexity = torch.exp(torch.tensor(eval_loss))
logger.info(f"total_loss:: {eval_loss}")
logger.info(f"total_perplexity:: {total_perplexity}")
with open(out_file_path, "w+", encoding="utf-8") as out_file:
for _, out_json in id_to_json_map.items():
out_file.write(out_json + "\n")
def main():
parser = HfArgumentParser(
(ModelArguments, DataTrainingArguments, TrainingArguments))
model_args, data_args, training_args = parser.parse_args_into_dataclasses()
if data_args.eval_data_file is None and training_args.do_eval:
raise ValueError(
"Cannot do evaluation without an evaluation data file. Either supply a file to --eval_data_file "
"or remove the --do_eval argument.")
if (os.path.exists(training_args.output_dir)
and os.listdir(training_args.output_dir) and training_args.do_train
and not training_args.overwrite_output_dir):
raise ValueError(
f"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
)
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO
if training_args.local_rank in [-1, 0] else logging.WARN,
)
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
training_args.local_rank,
training_args.device,
training_args.n_gpu,
bool(training_args.local_rank != -1),
training_args.fp16,
)
logger.info("Training/evaluation parameters %s", training_args)
# Set seed
set_seed(training_args.seed)
# Load pretrained model and tokenizer
#
# Distributed training:
# The .from_pretrained methods guarantee that only one local process can concurrently
# download model & vocab.
if model_args.config_name:
config = AutoConfig.from_pretrained(model_args.config_name,
cache_dir=model_args.cache_dir)
elif model_args.model_name_or_path:
config = AutoConfig.from_pretrained(model_args.model_name_or_path,
cache_dir=model_args.cache_dir)
else:
config = CONFIG_MAPPING[model_args.model_type]()
logger.warning(
"You are instantiating a new config instance from scratch.")
print("Config before overwrite max_position_embeddings:", config)
config.max_position_embeddings = 4096
config.num_hidden_layers = 6
config.num_attention_heads = 8
config.hidden_size = 512
config.intermediate_size = 2048
print("Config after overwrite max_position_embeddings:", config)
# if model_args.tokenizer_name:
# tokenizer = AutoTokenizer.from_pretrained(model_args.tokenizer_name, cache_dir=model_args.cache_dir)
# elif model_args.model_name_or_path:
# tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, cache_dir=model_args.cache_dir)
# else:
# raise ValueError(
# "You are instantiating a new tokenizer from scratch. This is not supported, but you can do it from another script, save it,"
# "and load it from here, using --tokenizer_name"
# )
logging.info("Loading tokenizer")
if model_args.tokenizer_name:
tokenizer = BertTokenizerFast(model_args.tokenizer_name,
clean_text=True,
lowercase=False,
strip_accents=True)
else:
raise ValueError("Specify tokenizer name")
logging.info("Loading model")
if model_args.model_name_or_path:
model = AutoModelWithLMHead.from_pretrained(
model_args.model_name_or_path,
from_tf=bool(".ckpt" in model_args.model_name_or_path),
config=config,
cache_dir=model_args.cache_dir,
)
else:
logger.info("Training new model from scratch")
model = AutoModelWithLMHead.from_config(config)
logging.info("Resizing embeddings")
model.resize_token_embeddings(len(tokenizer))
print(len(tokenizer.get_vocab()), len(tokenizer))
if config.model_type in ["bert", "roberta", "distilbert", "camembert"
] and not data_args.mlm:
raise ValueError(
"BERT and RoBERTa-like models do not have LM heads but masked LM heads. They must be run using the"
"--mlm flag (masked language modeling).")
# Get datasets
logging.info("Loading train dataset")
train_dataset = get_dataset(data_args) if training_args.do_train else None
logging.info("Loading eval dataset")
eval_dataset = (get_dataset(
data_args,
evaluate=True,
) if training_args.do_eval else None)
if config.model_type == "xlnet":
data_collator = DataCollatorForPermutationLanguageModeling(
tokenizer=tokenizer,
plm_probability=data_args.plm_probability,
max_span_length=data_args.max_span_length,
)
else:
data_collator = DataCollatorForLanguageModeling(
tokenizer=tokenizer,
mlm=data_args.mlm,
mlm_probability=data_args.mlm_probability,
)
# Initialize our Trainer
logging.info("Initializing trainer")
trainer = Trainer(
model=model,
args=training_args,
data_collator=data_collator,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
prediction_loss_only=True,
)
# Training
if training_args.do_train:
logging.info("Training")
model_path = (model_args.model_name_or_path
if model_args.model_name_or_path is not None
and os.path.isdir(model_args.model_name_or_path) else
None)
trainer.train(model_path=model_path)
trainer.save_model()
# For convenience, we also re-save the tokenizer to the same directory,
# so that you can share your model easily on huggingface.co/models =)
if trainer.is_world_master():
tokenizer.save_pretrained(training_args.output_dir)
# Evaluation
results = {}
if training_args.do_eval:
logger.info("*** Evaluate ***")
eval_output = trainer.evaluate()
perplexity = math.exp(eval_output["eval_loss"])
result = {"perplexity": perplexity}
output_eval_file = os.path.join(training_args.output_dir,
"eval_results_lm.txt")
if trainer.is_world_master():
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
results.update(result)
return results
def main(args):
if args.eval_data_file is None and args.do_eval:
raise ValueError(
"Cannot do evaluation without an evaluation data file. Either supply a file to --eval_data_file "
"or remove the --do_eval argument.")
if args.should_continue:
sorted_checkpoints = _sorted_checkpoints(args)
if len(sorted_checkpoints) == 0:
raise ValueError(
"Used --should_continue but no checkpoint was found in --ckpt_dir."
)
else:
args.model_name = sorted_checkpoints[-1]
if (os.path.exists(args.ckpt_dir) and os.listdir(args.ckpt_dir)
and args.do_train and not args.overwrite_ckpt_dir
and not args.should_continue):
raise ValueError(
"Output directory ({}) already exists and is not empty. Use --overwrite_ckpt_dir to overcome."
.format(args.ckpt_dir))
# Setup CUDA, GPU & distributed training
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
args.device = device
# Setup logging
os.makedirs(args.ckpt_dir, exist_ok=True)
log_path = os.path.join(
args.ckpt_dir, f'{datetime.now().strftime("%m%d_%H%M%S.%f")}.log')
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO,
handlers=[logging.FileHandler(log_path),
logging.StreamHandler()])
logger.warning(
"device: %s, 16-bits training: %s",
device,
args.fp16,
)
# Set seed
set_seed(args)
if args.config_name:
config = AutoConfig.from_pretrained(args.config_name)
elif args.model_name:
config = AutoConfig.from_pretrained(args.model_name)
else:
# When we release a pip version exposing CONFIG_MAPPING,
# we can do `config = CONFIG_MAPPING[args.model_type]()`.
raise ValueError(
"You are instantiating a new config instance from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --config_name")
if args.tokenizer_name:
tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_name)
elif args.model_name:
tokenizer = AutoTokenizer.from_pretrained(args.model_name)
else:
raise ValueError(
"You are instantiating a new tokenizer from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --tokenizer_name")
if args.max_seq_len <= 0:
args.max_seq_len = tokenizer.max_len
# Our input block size will be the max possible for the model
else:
args.max_seq_len = min(args.max_seq_len, tokenizer.max_len)
if args.model_name:
model = AutoModelWithLMHead.from_pretrained(args.model_name,
config=config)
else:
logger.info("Training new model from scratch")
model = AutoModelWithLMHead.from_config(config)
special_tokens_dict = {'sep_token': '<SEP>', 'cls_token': '<CLS>'}
"""
Input format:
bank<SEP>door<CLS>bank has door<EOS>
people<SEP>drive<CLS>person like driving<EOS>
"""
tokenizer.add_special_tokens(special_tokens_dict)
tokenizer.pad_token = '!' # with id 0
model.to(args.device)
logger.info("Training/evaluation parameters %s", args)
# Training
if args.do_train:
train_dataset = TextDataset(tokenizer,
args,
file_path=args.train_data_file)
train(args, train_dataset, model, tokenizer)
# Evaluation
results = {}
if args.do_eval:
checkpoints = [args.ckpt_dir]
if args.eval_all_checkpoints:
checkpoints = list(
os.path.dirname(c) for c in sorted(
glob.glob(args.ckpt_dir + "/**/" + WEIGHTS_NAME,
recursive=True)))
logging.getLogger("transformers.modeling_utils").setLevel(
logging.WARN) # Reduce logging
logger.info("Evaluate the following checkpoints: %s", checkpoints)
for checkpoint in checkpoints:
global_step = checkpoint.split(
"-")[-1] if len(checkpoints) > 1 else ""
prefix = checkpoint.split(
"/")[-1] if checkpoint.find("checkpoint") != -1 else ""
model = AutoModelWithLMHead.from_pretrained(checkpoint)
model.to(args.device)
result = evaluate(args, model, tokenizer, prefix=prefix)
result = dict(
(k + "_{}".format(global_step), v) for k, v in result.items())
results.update(result)
return results
def main():
## step 1: defining the argument.
parser = argparse.ArgumentParser()
# Required parameters
parser.add_argument("--train_data_file",
default=None,
type=str,
required=True,
help="The input training data file (a text file).")
# output_dir better be independent w.r.t name
parser.add_argument(
"--output_dir",
type=str,
required=True,
help=
"The output directory where the model predictions and checkpoints will be written.",
)
parser.add_argument(
"--model_type",
type=str,
help="The model architecture to be trained or fine-tuned.",
)
parser.set_defaults(model_type='bert')
# Other parameters
parser.add_argument(
"--eval_data_file",
default=None,
type=str,
help=
"An optional input evaluation data file to evaluate the perplexity on (a text file).",
)
parser.add_argument(
"--line_by_line",
action="store_true",
help=
"Whether distinct lines of text in the dataset are to be handled as distinct sequences.",
)
parser.add_argument(
"--should_continue",
action="store_true",
help="Whether to continue from latest checkpoint in output_dir")
parser.add_argument(
"--model_name_or_path",
default=None,
type=str,
help=
"The model checkpoint for weights initialization. Leave None if you want to train a model from scratch.",
)
parser.add_argument(
"--mlm",
action="store_true",
help=
"Train with masked-language modeling loss instead of language modeling."
)
parser.add_argument(
"--mlm_probability",
type=float,
default=0.15,
help="Ratio of tokens to mask for masked language modeling loss")
parser.set_defaults(mlm=True)
parser.add_argument(
"--config_name",
default=None,
type=str,
help=
"Optional pretrained config name or path if not the same as model_name_or_path. If both are None, initialize a new config.",
)
parser.add_argument(
"--tokenizer_name",
default=None,
type=str,
help=
"Optional pretrained tokenizer name or path if not the same as model_name_or_path. If both are None, initialize a new tokenizer.",
)
parser.add_argument(
"--cache_dir",
default=None,
type=str,
help=
"Optional directory to store the pre-trained models downloaded from s3 (instead of the default one)",
)
parser.add_argument(
"--block_size",
default=-1,
type=int,
help="Optional input sequence length after tokenization."
"The training dataset will be truncated in block of this size for training."
"Default to the model max input length for single sentence inputs (take into account special tokens).",
)
parser.add_argument("--do_train",
action="store_true",
help="Whether to run training.")
parser.add_argument("--do_eval",
action="store_true",
help="Whether to run eval on the dev set.")
parser.set_defaults(do_train=True)
parser.set_defaults(do_eval=True)
parser.add_argument("--per_gpu_train_batch_size",
default=4,
type=int,
help="Batch size per GPU/CPU for training.")
parser.add_argument("--per_gpu_eval_batch_size",
default=4,
type=int,
help="Batch size per GPU/CPU for evaluation.")
parser.add_argument(
"--gradient_accumulation_steps",
type=int,
default=1,
help=
"Number of updates steps to accumulate before performing a backward/update pass.",
)
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--weight_decay",
default=0.0,
type=float,
help="Weight decay if we apply some.")
parser.add_argument("--adam_epsilon",
default=1e-8,
type=float,
help="Epsilon for Adam optimizer.")
parser.add_argument("--max_grad_norm",
default=1.0,
type=float,
help="Max gradient norm.")
parser.add_argument("--num_train_epochs",
default=1.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--max_steps",
default=-1,
type=int,
help=
"If > 0: set total number of training steps to perform. Override num_train_epochs.",
)
parser.add_argument("--warmup_steps",
default=0,
type=int,
help="Linear warmup over warmup_steps.")
parser.add_argument("--logging_steps",
type=int,
default=1500,
help="Log every X updates steps.")
parser.add_argument(
"--eval_all_checkpoints",
action="store_true",
help=
"Evaluate all checkpoints starting with the same prefix as model_name_or_path ending and ending with step number",
)
parser.add_argument("--no_cuda",
action="store_true",
help="Avoid using CUDA when available")
parser.add_argument("--overwrite_output_dir",
action="store_true",
help="Overwrite the content of the output directory")
parser.set_defaults(overwrite_output_dir=True)
parser.add_argument(
"--overwrite_cache",
action="store_true",
help="Overwrite the cached training and evaluation sets")
parser.add_argument("--seed",
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
"--fp16",
action="store_true",
help=
"Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit",
)
parser.add_argument(
"--fp16_opt_level",
type=str,
default="O1",
help=
"For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
"See details at https://nvidia.github.io/apex/amp.html",
)
parser.add_argument("--debug", action="store_true", help="Debug mode")
parser.set_defaults(debug=False)
parser.add_argument("--local_rank",
type=int,
default=-1,
help="For distributed training: local_rank")
parser.add_argument("--server_ip",
type=str,
default="",
help="For distant debugging.")
parser.add_argument("--server_port",
type=str,
default="",
help="For distant debugging.")
args = parser.parse_args()
## step 2: to check the correctness of arguments.
if args.model_type in ["bert", "roberta", "distilbert", "camembert"
] and not args.mlm:
raise ValueError(
"BERT and RoBERTa-like models do not have LM heads but masked LM heads. They must be run using the --mlm "
"flag (masked language modeling).")
if args.eval_data_file is None and args.do_eval:
raise ValueError(
"Cannot do evaluation without an evaluation data file. Either supply a file to --eval_data_file "
"or remove the --do_eval argument.")
if args.should_continue:
previous_checkpoints = _get_previous_checkpoint(args)
if len(previous_checkpoints) == 0:
raise ValueError(
"Used --should_continue but no checkpoint was found in --output_dir."
)
else:
args.model_name_or_path = previous_checkpoints[0]
if (os.path.exists(args.output_dir) and os.listdir(args.output_dir)
and args.do_train and not args.overwrite_output_dir
and not args.should_continue):
raise ValueError(
"Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome."
.format(args.output_dir))
## step 3: setup various things: exper description, debug server, device and distributed training, and logging.
args.descr_string = "model-{}_lr-{}_maxepoch-{}_bs-{}_warmup-{}".format(
"SciBERT", args.learning_rate, args.num_train_epochs,
args.per_gpu_train_batch_size, args.warmup_steps)
# Setup the debug mode
if args.debug:
args.logging_steps = 2
args.max_steps = 20
# Setup distant debugging if needed
if args.server_ip and args.server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
print("Waiting for debugger attach")
ptvsd.enable_attach(address=(args.server_ip, args.server_port),
redirect_output=True)
ptvsd.wait_for_attach()
# Setup CUDA, GPU & distributed training
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
args.n_gpu = 0 if args.no_cuda else torch.cuda.device_count()
else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
torch.distributed.init_process_group(backend="nccl")
args.n_gpu = 1
args.device = device
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
args.local_rank,
device,
args.n_gpu,
bool(args.local_rank != -1),
args.fp16,
)
# Set seed
set_seed(args)
## step 4: loading the pretrained model and tokenizer.
# Load pretrained model and tokenizer
if args.local_rank not in [-1, 0]:
torch.distributed.barrier(
) # Barrier to make sure only the first process in distributed training download model & vocab
if args.config_name:
config = AutoConfig.from_pretrained(args.config_name,
cache_dir=args.cache_dir)
elif args.model_name_or_path:
config = AutoConfig.from_pretrained(args.model_name_or_path,
cache_dir=args.cache_dir)
else:
# When we release a pip version exposing CONFIG_MAPPING,
# we can do `config = CONFIG_MAPPING[args.model_type]()`.
raise ValueError(
"You are instantiating a new config instance from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --config_name")
if args.tokenizer_name:
tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_name,
cache_dir=args.cache_dir)
elif args.model_name_or_path:
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path,
cache_dir=args.cache_dir)
else:
raise ValueError(
"You are instantiating a new tokenizer from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --tokenizer_name")
if args.block_size <= 0:
args.block_size = tokenizer.max_len
# Our input block size will be the max possible for the model
else:
args.block_size = min(args.block_size, tokenizer.max_len)
if args.model_name_or_path:
model = AutoModelWithLMHead.from_pretrained(
args.model_name_or_path,
from_tf=bool(".ckpt" in args.model_name_or_path),
config=config,
cache_dir=args.cache_dir,
)
else:
logger.info("Training new model from scratch")
model = AutoModelWithLMHead.from_config(config)
model.to(args.device)
if args.local_rank == 0:
torch.distributed.barrier(
) # End of barrier to make sure only the first process in distributed training download model & vocab
logger.info("Training/evaluation parameters %s", args)
## step 5: train and save the model after the training
# Training
if args.do_train:
if args.local_rank not in [-1, 0]:
torch.distributed.barrier(
) # Barrier to make sure only the first process in distributed training process the dataset, and the others will use the cache
train_dataset = load_and_cache_examples(args,
tokenizer,
evaluate=False)
if args.local_rank == 0:
torch.distributed.barrier()
global_step, tr_loss = train(args, train_dataset, model, tokenizer)
logger.info(" global_step = %s, average loss = %s", global_step,
tr_loss)
def main():
# See all possible arguments in src/transformers/training_args.py
# or by passing the --help flag to this script.
# We now keep distinct sets of args, for a cleaner separation of concerns.
parser = HfArgumentParser(
(ModelArguments, DataTrainingArguments, TrainingArguments))
model_args, data_args, training_args = parser.parse_args_into_dataclasses()
training_args.do_train = True
# training_args.do_eval = True
data_args.mlm = True
if data_args.eval_data_file is None and training_args.do_eval:
raise ValueError(
"Cannot do evaluation without an evaluation data file. Either supply a file to --eval_data_file "
"or remove the --do_eval argument.")
if (os.path.exists(training_args.output_dir)
and os.listdir(training_args.output_dir) and training_args.do_train
and not training_args.overwrite_output_dir):
raise ValueError(
f"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
)
training_args.output_dir = os.environ["SM_OUTPUT_DATA_DIR"]
input_path = os.environ["SM_CHANNEL_TRAINING"]
folder = "wikitext-2-raw-v1"
cmd = "tar xzf %s -C %s" % (input_path + "/" + folder + ".tar.gz",
input_path)
assert os.system(cmd) == 0
data_folder = f"{input_path}/{folder}"
data_args.train_data_file = f"{data_folder}/wikitext-2-raw/{data_args.train_data_file}"
data_args.eval_data_file = f"{data_folder}/wikitext-2-raw/{data_args.eval_data_file}"
checkpoint_path = training_args.output_dir + "/checkpoints"
project_name = "language-modeling"
os.environ["WANDB_PROJECT"] = project_name
wandb.init(
project=project_name
) # TODO(tilo): is this really necessary? should be done by ML-library ( here transformers)
assert wandb.api.api_key is not None
assert is_wandb_available(
) # TODO(tilo): somehow I had issues, which seem to have resolved themselves
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO
if training_args.local_rank in [-1, 0] else logging.WARN,
)
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
training_args.local_rank,
training_args.device,
training_args.n_gpu,
bool(training_args.local_rank != -1),
training_args.fp16,
)
logger.info("Training/evaluation parameters %s", training_args)
# Set seed
set_seed(training_args.seed)
# Load pretrained model and tokenizer
#
# Distributed training:
# The .from_pretrained methods guarantee that only one local process can concurrently
# download model & vocab.
if model_args.config_name:
config = AutoConfig.from_pretrained(model_args.config_name,
cache_dir=model_args.cache_dir)
elif model_args.model_name_or_path:
config = AutoConfig.from_pretrained(model_args.model_name_or_path,
cache_dir=model_args.cache_dir)
else:
config = CONFIG_MAPPING[model_args.model_type]()
logger.warning(
"You are instantiating a new config instance from scratch.")
if model_args.tokenizer_name:
tokenizer = AutoTokenizer.from_pretrained(
model_args.tokenizer_name, cache_dir=model_args.cache_dir)
elif model_args.model_name_or_path:
tokenizer = AutoTokenizer.from_pretrained(
model_args.model_name_or_path, cache_dir=model_args.cache_dir)
else:
raise ValueError(
"You are instantiating a new tokenizer from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --tokenizer_name")
if model_args.model_name_or_path:
model = AutoModelWithLMHead.from_pretrained(
model_args.model_name_or_path,
from_tf=bool(".ckpt" in model_args.model_name_or_path),
config=config,
cache_dir=model_args.cache_dir,
)
else:
logger.info("Training new model from scratch")
model = AutoModelWithLMHead.from_config(config)
model.resize_token_embeddings(len(tokenizer))
if config.model_type in ["bert", "roberta", "distilbert", "camembert"
] and not data_args.mlm:
raise ValueError(
"BERT and RoBERTa-like models do not have LM heads but masked LM heads. They must be run using the"
"--mlm flag (masked language modeling).")
if data_args.block_size <= 0:
data_args.block_size = tokenizer.max_len
# Our input block size will be the max possible for the model
else:
data_args.block_size = min(data_args.block_size, tokenizer.max_len)
# Get datasets
train_dataset = (get_dataset(
data_args, tokenizer=tokenizer, cache_dir=model_args.cache_dir)
if training_args.do_train else None)
eval_dataset = (get_dataset(data_args,
tokenizer=tokenizer,
evaluate=True,
cache_dir=model_args.cache_dir)
if training_args.do_eval else None)
if config.model_type == "xlnet":
data_collator = DataCollatorForPermutationLanguageModeling(
tokenizer=tokenizer,
plm_probability=data_args.plm_probability,
max_span_length=data_args.max_span_length,
)
else:
data_collator = DataCollatorForLanguageModeling(
tokenizer=tokenizer,
mlm=data_args.mlm,
mlm_probability=data_args.mlm_probability)
# Initialize our Trainer
trainer = Trainer(
model=model,
args=training_args,
data_collator=data_collator,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
prediction_loss_only=True,
)
# Training
if training_args.do_train:
model_path = (model_args.model_name_or_path
if model_args.model_name_or_path is not None
and os.path.isdir(model_args.model_name_or_path) else
None)
trainer.train(model_path=model_path)
trainer.save_model()
# For convenience, we also re-save the tokenizer to the same directory,
# so that you can share your model easily on huggingface.co/models =)
if trainer.is_world_master():
tokenizer.save_pretrained(training_args.output_dir)
# Evaluation
results = {}
if training_args.do_eval:
logger.info("*** Evaluate ***")
eval_output = trainer.evaluate()
perplexity = math.exp(eval_output["eval_loss"])
result = {"perplexity": perplexity}
output_eval_file = os.path.join(training_args.output_dir,
"eval_results_lm.txt")
if trainer.is_world_master():
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
results.update(result)
return results
def main():
# See all possible arguments in src/transformers/training_args.py
# or by passing the --help flag to this script.
# We now keep distinct sets of args, for a cleaner separation of concerns.
parser = HfArgumentParser(
(ModelArguments, DataTrainingArguments, TrainingArguments))
model_args, data_args, training_args = parser.parse_args_into_dataclasses()
if data_args.eval_data_file is None and training_args.do_eval:
raise ValueError(
"Cannot do evaluation without an evaluation data file. Either supply a file to --eval_data_file "
"or remove the --do_eval argument.")
if (os.path.exists(training_args.output_dir)
and os.listdir(training_args.output_dir) and training_args.do_train
and not training_args.overwrite_output_dir):
raise ValueError(
f"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
)
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO
if training_args.local_rank in [-1, 0] else logging.WARN,
)
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
training_args.local_rank,
training_args.device,
training_args.n_gpu,
bool(training_args.local_rank != -1),
training_args.fp16,
)
logger.info("Training/evaluation parameters %s", training_args)
# Set seed
set_seed(training_args.seed)
# Load pretrained model and tokenizer
#
# Distributed training:
# The .from_pretrained methods guarantee that only one local process can concurrently
# download model & vocab.
if model_args.config_name:
config = AutoConfig.from_pretrained(model_args.config_name,
cache_dir=model_args.cache_dir)
elif model_args.model_name_or_path:
config = AutoConfig.from_pretrained(model_args.model_name_or_path,
cache_dir=model_args.cache_dir)
else:
config = CONFIG_MAPPING[model_args.model_type]()
logger.warning(
"You are instantiating a new config instance from scratch.")
if model_args.tokenizer_name:
tokenizer = AutoTokenizer.from_pretrained(
model_args.tokenizer_name, cache_dir=model_args.cache_dir)
elif model_args.model_name_or_path:
tokenizer = AutoTokenizer.from_pretrained(
model_args.model_name_or_path, cache_dir=model_args.cache_dir)
else:
raise ValueError(
"You are instantiating a new tokenizer from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --tokenizer_name")
if model_args.model_name_or_path:
model = AutoModelForPreTraining.from_pretrained(
model_args.model_name_or_path,
from_tf=bool(".ckpt" in model_args.model_name_or_path),
config=config,
cache_dir=model_args.cache_dir,
)
else:
logger.info("Training new model from scratch")
model = AutoModelWithLMHead.from_config(config)
model.resize_token_embeddings(len(tokenizer))
if config.model_type in ["bert", "roberta", "distilbert", "camembert"
] and not data_args.mlm:
raise ValueError(
"BERT and RoBERTa-like models do not have LM heads but masked LM heads. They must be run using the --mlm "
"flag (masked language modeling).")
if data_args.block_size <= 0:
data_args.block_size = tokenizer.max_len
# Our input block size will be the max possible for the model
else:
data_args.block_size = min(data_args.block_size, tokenizer.max_len)
# Get datasets
train_dataset = get_dataset(
data_args, tokenizer=tokenizer) if training_args.do_train else None
eval_dataset = get_dataset(
data_args, tokenizer=tokenizer,
evaluate=True) if training_args.do_eval else None
data_collator = DataCollatorForLanguageModelingAVSD(
tokenizer=tokenizer,
mlm=data_args.mlm,
mlm_probability=data_args.mlm_probability)
# Initialize our Trainer
trainer = Trainer(
model=model,
args=training_args,
data_collator=data_collator,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
#prediction_loss_only=True,
)
# Training
if training_args.do_train:
model_path = (model_args.model_name_or_path
if model_args.model_name_or_path is not None
and os.path.isdir(model_args.model_name_or_path) else
None)
trainer.train(model_path=model_path)
trainer.save_model()
# For convenience, we also re-save the tokenizer to the same directory,
# so that you can share your model easily on huggingface.co/models =)
if trainer.is_world_master():
tokenizer.save_pretrained(training_args.output_dir)
# Evaluation
results = {}
avsd_eval = True
outputs = []
if avsd_eval:
outputs = []
model = trainer.model
data_loader = trainer.get_eval_dataloader(eval_dataset)
for batch in tqdm(data_loader):
for k, v in batch.items():
batch[k] = v.to(training_args.device)
with torch.no_grad():
output = model(**batch)
nsp_scores = output[1]
#nsp_probs = F.softmax(nsp_scores, dim=1)
nsp_scores = nsp_scores[:, 0].detach().cpu().tolist()
outputs.extend(nsp_scores)
results['avsd_train_set'] = outputs
json.dump(
results,
open(
'/home/halamri/summer2020/avsd-transofrmers/mlmAVSD/bertPredicitonResults_June15_corrected.txt',
'w'))
print('Done....')
'''
def main():
parser = argparse.ArgumentParser()
# Required parameters
parser.add_argument("--data_file",
default=None,
type=str,
required=True,
help="The input data file.")
parser.add_argument(
"--output_dir",
type=str,
required=True,
help=
"The output directory where the model predictions and checkpoints will be written.",
)
parser.add_argument(
"--model_type",
type=str,
required=True,
help="The model architecture to be trained or fine-tuned.",
)
# Other parameters
parser.add_argument(
"--eval_data_file",
default=None,
type=str,
help=
"An optional input evaluation data file to evaluate the perplexity on (a text file).",
)
parser.add_argument(
"--line_by_line",
action="store_true",
help=
"Whether distinct lines of text in the dataset are to be handled as distinct sequences.",
)
parser.add_argument(
"--should_continue",
action="store_true",
help="Whether to continue from latest checkpoint in output_dir")
parser.add_argument(
"--model_name_or_path",
default=None,
type=str,
help=
"The model checkpoint for weights initialization. Leave None if you want to train a model from scratch.",
)
parser.add_argument("--loss_target", type=str, help="")
parser.add_argument(
"--mlm",
action="store_true",
help=
"Train with masked-language modeling loss instead of language modeling."
)
parser.add_argument(
"--mlm_probability",
type=float,
default=0.15,
help="Ratio of tokens to mask for masked language modeling loss")
parser.add_argument(
"--config_name",
default=None,
type=str,
help=
"Optional pretrained config name or path if not the same as model_name_or_path. If both are None, initialize a new config.",
)
parser.add_argument(
"--tokenizer_name",
default=None,
type=str,
help=
"Optional pretrained tokenizer name or path if not the same as model_name_or_path. If both are None, initialize a new tokenizer.",
)
parser.add_argument(
"--cache_dir",
default=None,
type=str,
help=
"Optional directory to store the pre-trained models downloaded from s3 (instead of the default one)",
)
parser.add_argument(
"--block_size",
default=-1,
type=int,
help="Optional input sequence length after tokenization."
"The training dataset will be truncated in block of this size for training."
"Default to the model max input length for single sentence inputs (take into account special tokens).",
)
parser.add_argument("--do_train",
action="store_true",
help="Whether to run training.")
parser.add_argument("--do_eval",
action="store_true",
help="Whether to run eval on the dev set.")
parser.add_argument(
"--evaluate_during_training",
action="store_true",
help="Run evaluation during training at each logging step.")
parser.add_argument("--per_gpu_train_batch_size",
default=4,
type=int,
help="Batch size per GPU/CPU for training.")
parser.add_argument("--per_gpu_eval_batch_size",
default=4,
type=int,
help="Batch size per GPU/CPU for evaluation.")
parser.add_argument(
"--gradient_accumulation_steps",
type=int,
default=1,
help=
"Number of updates steps to accumulate before performing a backward/update pass.",
)
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--weight_decay",
default=0.0,
type=float,
help="Weight decay if we apply some.")
parser.add_argument("--adam_epsilon",
default=1e-8,
type=float,
help="Epsilon for Adam optimizer.")
parser.add_argument("--max_grad_norm",
default=1.0,
type=float,
help="Max gradient norm.")
parser.add_argument("--weighted_loss",
default=[1.0, 1.0],
nargs='*',
help="")
parser.add_argument("--debias_layer",
default=all,
type=str,
choices=['all', 'first', 'last'],
help="")
parser.add_argument("--num_train_epochs",
default=1.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--max_steps",
default=-1,
type=int,
help=
"If > 0: set total number of training steps to perform. Override num_train_epochs.",
)
parser.add_argument("--warmup_steps",
default=0,
type=int,
help="Linear warmup over warmup_steps.")
parser.add_argument("--square_loss", action='store_true', help="")
parser.add_argument("--token_loss", action='store_true', help="")
parser.add_argument("--logging_steps",
type=int,
default=500,
help="Log every X updates steps.")
parser.add_argument("--save_steps",
type=int,
default=500,
help="Save checkpoint every X updates steps.")
parser.add_argument(
"--save_total_limit",
type=int,
default=None,
help=
"Limit the total amount of checkpoints, delete the older checkpoints in the output_dir, does not delete by default",
)
parser.add_argument(
"--eval_all_checkpoints",
action="store_true",
help=
"Evaluate all checkpoints starting with the same prefix as model_name_or_path ending and ending with step number",
)
parser.add_argument("--no_cuda",
action="store_true",
help="Avoid using CUDA when available")
parser.add_argument("--overwrite_output_dir",
action="store_true",
help="Overwrite the content of the output directory")
parser.add_argument(
"--overwrite_cache",
action="store_true",
help="Overwrite the cached training and evaluation sets")
parser.add_argument("--seed",
type=int,
default=42,
help="random seed for initialization")
parser.add_argument("--dev_data_size", type=int, default=1000, help="")
parser.add_argument(
"--fp16",
action="store_true",
help=
"Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit",
)
parser.add_argument(
"--fp16_opt_level",
type=str,
default="O1",
help=
"For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
"See details at https://nvidia.github.io/apex/amp.html",
)
tp = lambda x: list(x.split(','))
parser.add_argument('--exclusion_list', type=tp, default=[], help='')
parser.add_argument("--local_rank",
type=int,
default=-1,
help="For distributed training: local_rank")
parser.add_argument("--server_ip",
type=str,
default="",
help="For distant debugging.")
parser.add_argument("--server_port",
type=str,
default="",
help="For distant debugging.")
args = parser.parse_args()
'''
if args.model_type in ["bert", "roberta", "distilbert", "camembert"] and not args.mlm:
raise ValueError(
"BERT and RoBERTa-like models do not have LM heads but masked LM heads. They must be run using the --mlm "
"flag (masked language modeling)."
)
'''
if args.should_continue:
sorted_checkpoints = _sorted_checkpoints(args)
if len(sorted_checkpoints) == 0:
raise ValueError(
"Used --should_continue but no checkpoint was found in --output_dir."
)
else:
args.model_name_or_path = sorted_checkpoints[-1]
if (os.path.exists(args.output_dir) and os.listdir(args.output_dir)
and args.do_train and not args.overwrite_output_dir
and not args.should_continue):
raise ValueError(
"Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome."
.format(args.output_dir))
# Setup distant debugging if needed
if args.server_ip and args.server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
print("Waiting for debugger attach")
ptvsd.enable_attach(address=(args.server_ip, args.server_port),
redirect_output=True)
ptvsd.wait_for_attach()
# Setup CUDA, GPU & distributed training
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
args.n_gpu = 0 if args.no_cuda else torch.cuda.device_count()
else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
torch.distributed.init_process_group(backend="nccl")
args.n_gpu = 1
args.device = device
# Set seed
set_seed(args)
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN,
)
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
args.local_rank,
device,
args.n_gpu,
bool(args.local_rank != -1),
args.fp16,
)
# Load pretrained model and tokenizer
if args.local_rank not in [-1, 0]:
torch.distributed.barrier(
) # Barrier to make sure only the first process in distributed training download model & vocab
if args.config_name:
config = AutoConfig.from_pretrained(args.config_name,
cache_dir=args.cache_dir)
elif args.model_name_or_path:
config = AutoConfig.from_pretrained(args.model_name_or_path,
cache_dir=args.cache_dir)
else:
# When we release a pip version exposing CONFIG_MAPPING,
# we can do `config = CONFIG_MAPPING[args.model_type]()`.
raise ValueError(
"You are instantiating a new config instance from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --config_name")
config.output_hidden_states = 'true' # 全層の隠れ層を取得する
#config.output_attentions = 'true' # 全層のアテンションを取得する
if args.tokenizer_name:
tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_name,
cache_dir=args.cache_dir)
elif args.model_name_or_path:
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path,
cache_dir=args.cache_dir)
else:
raise ValueError(
"You are instantiating a new tokenizer from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --tokenizer_name")
if args.block_size <= 0:
args.block_size = tokenizer.max_len
# Our input block size will be the max possible for the model
else:
args.block_size = min(args.block_size, tokenizer.max_len)
if args.model_name_or_path:
model = AutoModelWithLMHead.from_pretrained(
args.model_name_or_path,
from_tf=bool(".ckpt" in args.model_name_or_path),
config=config,
cache_dir=args.cache_dir,
)
original_model = AutoModelWithLMHead.from_pretrained(
args.model_name_or_path,
from_tf=bool(".ckpt" in args.model_name_or_path),
config=config,
cache_dir=args.cache_dir,
)
else:
logger.info("Training new model from scratch")
model = AutoModelWithLMHead.from_config(config)
original_model = AutoModelWithLMHead.from_config(config)
# GPT-2 and GPT do not have pad.
if tokenizer._pad_token is None:
tokenizer.add_special_tokens({'pad_token': '<pad>'})
model.resize_token_embeddings(len(tokenizer))
original_model.resize_token_embeddings(len(tokenizer))
model.to(args.device)
original_model.to(args.device)
if args.local_rank == 0:
torch.distributed.barrier(
) # End of barrier to make sure only the first process in distributed training download model & vocab
logger.info("Training/evaluation parameters %s", args)
data = torch.load(args.data_file)
attributes_examples = data['attributes_examples']
attributes_labels = data['attributes_labels']
neutral_examples = data['neutral_examples']
if 'neutral_labels' in data:
neutral_labels = data['neutral_labels']
splited_data = split_data(attributes_examples, attributes_labels,
neutral_examples, neutral_labels, args)
else:
splited_data = split_data(attributes_examples, attributes_labels,
neutral_examples, None, args)
datasets = load_and_cache_examples(splited_data, args, tokenizer)
if args.local_rank not in [-1, 0]:
torch.distributed.barrier(
) # Barrier to make sure only the first process in distributed training process the dataset, and the others will use the cache
if args.local_rank == 0:
torch.distributed.barrier()
train(args, splited_data, datasets, model, original_model, tokenizer)
def process(model_args, data_args, training_args):
if data_args.eval_data_file is None and training_args.do_eval:
raise ValueError(
"Cannot do evaluation without an evaluation data file. Either supply a file to --eval_data_file "
"or remove the --do_eval argument."
)
if (
os.path.exists(training_args.output_dir)
and os.listdir(training_args.output_dir)
and training_args.do_train
and not training_args.overwrite_output_dir
):
raise ValueError(
f"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
)
# Set seed
set_seed(training_args.seed)
if model_args.config_name:
config = AutoConfig.from_pretrained(model_args.config_name, cache_dir=model_args.cache_dir)
elif model_args.model_name_or_path:
config = AutoConfig.from_pretrained(model_args.model_name_or_path, cache_dir=model_args.cache_dir)
else:
config = CONFIG_MAPPING[model_args.model_type]()
if model_args.tokenizer_name:
tokenizer = AutoTokenizer.from_pretrained(model_args.tokenizer_name, cache_dir=model_args.cache_dir)
elif model_args.model_name_or_path:
tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, cache_dir=model_args.cache_dir)
else:
raise ValueError(
"You are instantiating a new tokenizer from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --tokenizer_name"
)
if model_args.model_name_or_path:
model = AutoModelWithLMHead.from_pretrained(
model_args.model_name_or_path,
from_tf=bool(".ckpt" in model_args.model_name_or_path),
config=config,
cache_dir=model_args.cache_dir,
)
else:
model = AutoModelWithLMHead.from_config(config)
model.resize_token_embeddings(len(tokenizer))
if config.model_type in ["bert", "roberta", "distilbert", "camembert"] and not data_args.mlm:
raise ValueError(
"BERT and RoBERTa-like models do not have LM heads but masked LM heads. They must be run using the --mlm "
"flag (masked language modeling)."
)
if data_args.block_size <= 0:
data_args.block_size = tokenizer.max_len
# Our input block size will be the max possible for the model
else:
data_args.block_size = min(data_args.block_size, tokenizer.max_len)
# Get datasets
train_dataset = get_dataset(data_args, tokenizer=tokenizer) if training_args.do_train else None
eval_dataset = get_dataset(data_args, tokenizer=tokenizer, evaluate=True) if training_args.do_eval else None
data_collator = DataCollatorForLanguageModeling(
tokenizer=tokenizer, mlm=data_args.mlm, mlm_probability=data_args.mlm_probability
)
# Initialize our Trainer
trainer = Trainer(
model=model,
args=training_args,
data_collator=data_collator,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
prediction_loss_only=True,
)
# Training
if training_args.do_train:
model_path = (
model_args.model_name_or_path
if model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path)
else None
)
trainer.train(model_path=model_path)
trainer.save_model()
# For convenience, we also re-save the tokenizer to the same directory,
# so that you can share your model easily on huggingface.co/models =)
if trainer.is_world_master():
tokenizer.save_pretrained(training_args.output_dir)
# Evaluation
results = {}
if training_args.do_eval:
eval_output = trainer.evaluate()
perplexity = math.exp(eval_output["eval_loss"])
result = {"perplexity": perplexity}
output_eval_file = os.path.join(training_args.output_dir, "eval_results_lm.txt")
if trainer.is_world_master():
with open(output_eval_file, "w") as writer:
for key in sorted(result.keys()):
writer.write("%s = %s\n" % (key, str(result[key])))
results.update(result)
return results
def main():
parser = argparse.ArgumentParser(
'Train or fine-tune a GPT-2 model using casual language (CLM) loss.')
parser.add_argument('train_dataset',
type=Path,
help='The preprocessed training dataset file.')
parser.add_argument('--eval-dataset',
type=Path,
help='The preprocessed evalutation dataset file.')
parser.add_argument(
'--line-by-line',
action='store_true',
help=
'Indicates whether distinct lines of text in the input dataset are to be handled as '
+
'separate input sequences (i.e. whether to add the BOS and EOS tokens to each line).'
)
parser.add_argument('--mlm',
dest='use_masked_loss',
action='store_true',
help='Train with masked-language modelling ' +
'(CLM) loss rather than casual language model loss.')
parser.add_argument('--mlm-probability',
type=float,
default=0.15,
help='Ratio of tokens to mask for MLM loss.')
parser.add_argument(
'--block-size',
type=int,
default=-1,
help='Optional input sequence length after tokenization. ' +
'The training dataset will be truncated in blocks of this size. Defaults to -1, meaning that '
+
'the maximum input sequence length of the model for single sentence inputs will be used.'
)
parser.add_argument(
'--overwrite-cache',
action='store_true',
help='Overwrite the cached training and evaluation sets.')
parser.add_argument(
'--model',
type=str,
default='gpt2',
dest='model_name_or_path',
help=
'The model checkpoint for weights initialization (i.e. a pretrained model). '
+ 'Leave as None to train a model from scratch.')
parser.add_argument('--model-type',
type=str,
default='gpt2',
help='The model architecture. Defaults to GPT2.')
parser.add_argument('--outdir',
type=Path,
default=Path('./output'),
help='The directory to save model checkpoints.')
parser.add_argument('--overwrite-outdir',
action='store_true',
help='Overwrrite the output directory.')
parser.add_argument('--logdir',
type=Path,
default=None,
help='The Tensorboard logging directory.')
parser.add_argument(
'--restore',
dest='restore_checkpoint',
action='store_true',
help='Whether to resume training from the lastest checkpoint.')
parser.add_argument('--cache-dir',
type=Path,
default=None,
help='The location to store pretrained models.')
parser.add_argument('--do-train',
action='store_true',
help='Whether to run training.')
parser.add_argument(
'--do-eval',
action='store_true',
help='Whether to evaluate the model on the evaluation set.')
parser.add_argument(
'--eval-during-training',
action='store_true',
help='Whether to run evaluation during training at each logging step.')
parser.add_argument(
'--eval-all-checkpoints',
action='store_true',
help=
'Evaluate all checkpoints starting with the same prefix as the input model checkpoint.'
)
parser.add_argument(
'--model-config',
default=None,
type=str,
help='Optional model config name or path if not the same as the model '
+
'checkpoint path. If both are None, a new config will be initialized.')
parser.add_argument(
'--tokenizer',
default=None,
type=str,
help=
'Optional pretrained tokenizer name or path if not the same as the model '
+
'checkpoint path. If both are None, a new tokenizer will be initialized.'
)
parser.add_argument(
'--train-batch-size',
'--per-device-train-batch-size',
dest='per_device_train_batch_size',
default=8,
type=int,
help='Batch size per device (i.e. GPU, CPU, or TPU) while training.')
parser.add_argument(
'--eval-batch-size',
'--per-device-eval-batch-size',
dest='per_device_eval_batch_size',
default=8,
type=int,
help='Batch size per device (i.e. GPU, CPU, or TPU) while evaluationg.'
)
parser.add_argument(
'--gradient-accumulation-steps',
type=int,
default=1,
help=
'Number of steps to accumulate before performing a backpropagation pass.'
)
parser.add_argument(
'--gradient-checkpointing',
action='store_true',
help=
'If True, use gradient checkpointing to save memory at the expense of slower backward pass.'
)
parser.add_argument(
'--learning-rate',
type=float,
default=5e-5,
help='The initial learning rate for the Adam optimizer.')
parser.add_argument('--weight-decay',
type=float,
default=0,
help='The rate at which weights decay.')
parser.add_argument('--epsilon',
type=float,
default=1e-8,
help='Epsilon for Adam optimizer.')
parser.add_argument('--max-gradient-norm',
type=float,
default=1.0,
help='Maximum gradient norm.')
parser.add_argument('--epochs',
type=float,
default=10.0,
help='Total number of training epochs to perform.')
parser.add_argument('--global-steps',
type=int,
default=-1,
help='The number of training steps to perform. ' +
'Overrides epochs parameter if non-negative value.')
parser.add_argument(
'--lr-warmup-steps',
type=int,
default=0,
help=
'The duration, in training steps, of the linear warmup on the learning rate.'
)
parser.add_argument(
'--log-frequency',
type=int,
default=500,
help=
'The frequency, in training steps, at which the model metrics are logged.'
)
parser.add_argument(
'--log-first-step',
action='store_true',
help='Log and evaluate the first global step. Defaults to False.')
parser.add_argument(
'--save-frequency',
type=int,
default=500,
help=
'The frequency, in training steps, at which the model checkpoint is saved.'
)
parser.add_argument(
'--max-checkpoints',
type=int,
default=None,
help=
'The maximum number of checkpoints to keep before deleting older ones. '
+ 'A negative or None value means that there is no limit.')
parser.add_argument(
'--no-cuda',
action='store_true',
help='Disable CUDA devices even when they are available.')
parser.add_argument('--fp16',
action='store_true',
help='Use 16-bit (mixed) precision floats.')
parser.add_argument(
'--fp16-opt-level',
type=str,
default='O1',
help=
'Apex AMP optimization level. See https://nvidia.github.io/apex/amp.html.'
)
parser.add_argument(
'--local-rank',
type=int,
default=-1,
help='The rank of the process; for distributed training. ' +
'A value of -1 means no distributed training.')
parser.add_argument('--debug-server-ip',
type=str,
default='',
help='The IP of the PTVSD server.')
parser.add_argument('--debug-server-port',
type=str,
default='',
help='The port of the PTVSD server.')
parser.add_argument(
'--special-tokens',
type=Path,
default=None,
help='A JSON file containing a dictionary of special tokens. ' +
'If not specified, the default special tokens are used (<|bos|>, <|eos|>, <|pad|>, and <|eq_tok|>).'
)
parser.add_argument('--seed',
type=int,
default=None,
help='The seed of the random engine.')
# We support both --tpu--num-cores and --tpu_num_cores for compatibility
# with the XLA launcher script, which injects the --tpu_num_cores argument.
parser.add_argument('--tpu-num-cores',
'--tpu_num_cores',
type=int,
default=None,
help='Number of TPU cores.')
parser.add_argument(
'--tpu-metrics-debug',
action='store_true',
help='Whether to print TPU debug metrics. Defaults to False.')
parser.add_argument(
'--dataloader-drop-last',
action='store_true',
help='Drop the last incomplete ' +
'batch if it is not divisible by the batch size. Defaults to False.')
args = parser.parse_args()
if args.logdir is None:
args.logdir = args.outdir / 'logs'
if args.model_type in ['bert', 'roberta', 'distilbert', 'camembert'
] and not args.use_masked_loss:
raise ValueError(
'BERT and RoBERTa-like models require masked language model heads. '
'They must be run with masked language modelliing, using the --mlm flag.'
)
if args.eval_dataset is None and args.do_eval:
raise ValueError(
'Cannot evaluate model without an evaluation dataset. '
'Either supply an evaluation dataset or remove the --do-eval flag.'
)
if args.restore_checkpoint:
checkpoints = get_checkpoints(args.outdir)
if len(checkpoints) == 0:
raise ValueError(
'Cannot restore the latest checkpoint: no checkpoint was found in the output directory (\'\').' \
.format(args.outdir)
)
else:
args.model_name_or_path = str(checkpoints[-1])
elif args.outdir.exists() and any(args.outdir.iterdir(
)) and args.do_train and not args.overwrite_outdir:
raise ValueError(
'Output directory (\'{}\') already exists and is non-empty. Use --overwrite-outdir to disable this.' \
.format(args.outdir)
)
# Setup PTVSD (Python Tools for Visual Studio Debugging) server
if args.debug_server_ip and args.debug_server_port:
import ptvsd
print('Waiting for PTVSD server to attach...')
ptvsd.enable_attach(address=(args.debug_server_ip,
args.debug_server_port),
redirect_output=True)
ptvsd.wait_for_attach()
# Setup CUDA, GPU, and distributed training
if args.local_rank == -1 or args.no_cuda:
args.device = torch.device('cuda' if torch.cuda.is_available()
and not args.no_cuda else 'cpu')
args.n_gpu = 0 if args.no_cuda else 1
else:
torch.cuda.set_device(args.local_rank)
args.device = torch.device('cuda', args.local_rank)
torch.distributed.init_process_group(backend='nccl')
args.n_gpu = 1
# Setup logging
logging.basicConfig(format='%(levelname)s: %(message)s',
datefmt='%m/%d/%Y %H:%M:%S')
logging.info(
'Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bit floating-point precision: %s',
args.local_rank,
args.device,
args.n_gpu,
bool(args.local_rank != -1),
args.fp16,
)
# Set seed
if args.seed is None:
# We use the current time as the seed rather than letting numpy seed
# since we want to achieve consistent results across sessions.
# Source: https://stackoverflow.com/a/45573061/7614083
t = int(time.time() * 1000.0)
args.seed = ((t & 0xff000000) >> 24) + ((t & 0x00ff0000) >> 8) + (
(t & 0x0000ff00) << 8) + ((t & 0x000000ff) << 24)
set_seed(args.seed)
if args.model_config:
config = AutoConfig.from_pretrained(args.model_config,
cache_dir=args.cache_dir)
elif args.model_name_or_path:
config = AutoConfig.from_pretrained(args.model_name_or_path,
cache_dir=args.cache_dir)
else:
config = CONFIG_MAPPING[args.model_type]()
logging.warning(
'No config found: creating a new config instance from scatch.')
# Inject gradient checkpoint value into config
if hasattr(config, 'gradient_checkpointing'):
config.gradient_checkpointing = args.gradient_checkpointing
else:
logging.warning(
'Config does not have the \'gradient_checkpointing\' attribute. '
'Gradient checkpointing will NOT be used.')
if args.tokenizer:
tokenizer = AutoTokenizer.from_pretrained(args.tokenizer,
cache_dir=args.cache_dir)
elif args.model_name_or_path:
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path,
cache_dir=args.cache_dir)
else:
raise ValueError(
'Instantiating a new tokenizer from scratch is not support; however, it can be done from another script.'
'Use the --tokenizer command line argument, providing it with the location of the script, to load the tokenizer.'
)
if args.model_name_or_path:
model = AutoModelWithLMHead.from_pretrained(
args.model_name_or_path,
from_tf=bool('.ckpt' in args.model_name_or_path),
config=config,
cache_dir=args.cache_dir)
else:
logging.info('Training {} model from scratch'.format(args.model_type))
model = AutoModelWithLMHead.from_config(config)
if args.block_size <= 0:
args.block_size = tokenizer.max_len
else:
args.block_size = min(args.block_size, tokenizer.max_len)
# Add special tokens and resize the model
logging.info(
'Initializing tokenizer with special tokens and resizing the model\'s token embeddings.'
)
tokenizer.add_special_tokens(get_special_tokens(args.special_tokens))
model.resize_token_embeddings(len(tokenizer))
# Load datasets
train_dataset = get_dataset(
args.train_dataset,
tokenizer,
args.block_size,
line_by_line=args.line_by_line,
overwrite_cache=args.overwrite_cache) if args.do_train else None
eval_dataset = get_dataset(
args.eval_dataset,
tokenizer,
args.block_size,
line_by_line=args.line_by_line,
overwrite_cache=args.overwrite_cache) if args.do_eval else None
data_collator = DataCollatorForLanguageModeling(
tokenizer=tokenizer,
mlm=args.use_masked_loss,
mlm_probability=args.mlm_probability)
# Project arguments to a TrainingArguments class.
training_args = TrainingArguments(
output_dir=str(args.outdir.absolute()),
overwrite_output_dir=args.overwrite_outdir,
do_train=args.do_train,
do_eval=args.do_eval,
evaluate_during_training=args.eval_during_training,
per_device_train_batch_size=args.per_device_train_batch_size,
per_device_eval_batch_size=args.per_device_eval_batch_size,
gradient_accumulation_steps=args.gradient_accumulation_steps,
learning_rate=args.learning_rate,
weight_decay=args.weight_decay,
adam_epsilon=args.epsilon,
max_grad_norm=args.max_gradient_norm,
num_train_epochs=args.epochs,
max_steps=args.global_steps,
warmup_steps=args.lr_warmup_steps,
logging_dir=str(args.logdir.absolute()),
logging_first_step=args.log_first_step,
logging_steps=args.log_frequency,
save_steps=args.save_frequency,
save_total_limit=args.max_checkpoints,
no_cuda=args.no_cuda,
seed=args.seed,
fp16=args.fp16,
fp16_opt_level=args.fp16_opt_level,
local_rank=args.local_rank,
tpu_num_cores=args.tpu_num_cores,
tpu_metrics_debug=args.tpu_metrics_debug)
# For some reason, the TrainingArguments __init__ method does not
# recognize dataloader_drop_last as a valid kwarg (it's possibly not implemented?).
# So, as a temporary workaround, we just set the field directly.
training_args.dataloader_drop_last = args.dataloader_drop_last
def _on_save_model(trainer, output_dir):
tokenizer.save_pretrained(output_dir)
trainer = Trainer(
model=model,
args=training_args,
data_collator=data_collator,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
prediction_loss_only=True,
# Callback to save tokenizer when checkpointing...
on_save_model=_on_save_model)
if args.do_train:
model_path = (args.model_name_or_path
if args.model_name_or_path is not None
and Path(args.model_name_or_path).is_dir() else None)
trainer.train(model_path=model_path)
trainer.save_model()
if trainer.is_world_master():
tokenizer.save_pretrained(args.outdir)
if args.do_eval:
checkpoints = [args.outdir]
if args.eval_all_checkpoints:
checkpoints = list(args.outdir.glob('**/{}'.format(WEIGHTS_NAME)))
logging.getLogger('transformers.modelling_utils').setLevel(
logging.WARN)
logging.info(
'Evaluating the following checkpoints: {}'.format(checkpoints))
for checkpoint in checkpoints:
model = AutoModelWithLMHead.from_pretrained(
str(checkpoint.absolute()))
# Recreate trainer with model from checkpoint
trainer = Trainer(model=model,
args=training_args,
data_collator=data_collator,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
prediction_loss_only=True)
eval_output = trainer.evaluate()
perplexity = math.exp(eval_output['eval_loss'])
result = {
'perplexity': perplexity,
'loss': eval_output['eval_loss']
}
prefix = str(checkpoint).split(
'/')[-1] if str(checkpoint).find('checkpoint') != -1 else ''
logging.info('***** Evaluation results {} *****'.format(prefix))
if trainer.is_world_master():
with open(args.outdir / (prefix + 'eval_results.txt'),
'w+') as file:
for key in sorted(result.keys()):
logging.info(' {}: {}'.format(key, str(result[key])))
file.write('{}={}\n'.format(key, str(result[key])))
def main():
args = get_args()
if args.model_type in ["bert", "roberta", "distilbert", "camembert"] and not \
(args.mlm or args.token_discrimination or args.mask_token_discrimination):
raise ValueError(
"BERT and RoBERTa-like models do not have LM heads but masked LM heads. They must be run using the --mlm "
"flag (masked language modeling).")
if args.eval_data_file is None and args.do_eval:
raise ValueError(
"Cannot do evaluation without an evaluation data file. Either supply a file to --eval_data_file "
"or remove the --do_eval argument.")
if args.should_continue:
sorted_checkpoints = _sorted_checkpoints(args)
if len(sorted_checkpoints) == 0:
raise ValueError(
"Used --should_continue but no checkpoint was found in --output_dir."
)
else:
args.model_name_or_path = sorted_checkpoints[-1]
if (os.path.exists(args.output_dir) and os.listdir(args.output_dir)
and args.do_train and not args.overwrite_output_dir):
raise ValueError(
"Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome."
.format(args.output_dir))
# Setup distant debugging if needed
if args.server_ip and args.server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
print("Waiting for debugger attach")
ptvsd.enable_attach(address=(args.server_ip, args.server_port),
redirect_output=True)
ptvsd.wait_for_attach()
# Setup CUDA, GPU & distributed training
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
args.n_gpu = 0 if args.no_cuda else torch.cuda.device_count()
else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
torch.distributed.init_process_group(backend="nccl")
args.n_gpu = 1
args.device = device
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN,
)
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
args.local_rank,
device,
args.n_gpu,
bool(args.local_rank != -1),
args.fp16,
)
# Set seed
set_seed(args)
# Load pretrained model and tokenizer
if args.local_rank not in [-1, 0]:
torch.distributed.barrier(
) # Barrier to make sure only the first process in distributed training download model & vocab
if args.config_name:
config = AutoConfig.from_pretrained(args.config_name,
cache_dir=args.cache_dir)
elif args.model_name_or_path:
config = AutoConfig.from_pretrained(args.model_name_or_path,
cache_dir=args.cache_dir)
else:
# When we release a pip version exposing CONFIG_MAPPING,
# we can do `config = CONFIG_MAPPING[args.model_type]()`.
raise ValueError(
"You are instantiating a new config instance from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --config_name")
if args.tokenizer_name:
tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_name,
cache_dir=args.cache_dir)
elif args.model_name_or_path:
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path,
cache_dir=args.cache_dir)
else:
raise ValueError(
"You are instantiating a new tokenizer from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --tokenizer_name")
if args.block_size <= 0:
args.block_size = tokenizer.max_len
# Our input block size will be the max possible for the model
else:
args.block_size = min(args.block_size, tokenizer.max_len)
if args.model_name_or_path and (args.token_discrimination
or args.mask_token_discrimination):
model = RobertaForTokenDiscrimination.from_pretrained(
args.model_name_or_path,
from_tf=bool(".ckpt" in args.model_name_or_path),
config=config,
cache_dir=args.cache_dir,
)
elif args.model_name_or_path and args.mlm:
model = AutoModelWithLMHead.from_pretrained(
args.model_name_or_path,
from_tf=bool(".ckpt" in args.model_name_or_path),
config=config,
cache_dir=args.cache_dir,
)
else:
logger.info("Training new model from scratch")
model = AutoModelWithLMHead.from_config(config)
model.to(args.device)
if args.local_rank == 0:
torch.distributed.barrier() # End of barrier
# to make sure only the first process
# in distributed training download model & vocab
logger.info("Training/evaluation parameters %s", args)
# Training
if args.do_train:
if args.local_rank not in [-1, 0]:
torch.distributed.barrier(
) # Barrier to make sure only the first process in distributed training process the dataset, and the others will use the cache
train_dataset = load_and_cache_examples(args,
tokenizer,
evaluate=False)
if args.local_rank == 0:
torch.distributed.barrier()
global_step, tr_loss = train(args, train_dataset, model, tokenizer)
logger.info(" global_step = %s, average loss = %s", global_step,
tr_loss)
# Saving best-practices: if you use save_pretrained for the model and tokenizer,
# you can reload them using from_pretrained()
if args.do_train and (args.local_rank == -1
or torch.distributed.get_rank() == 0):
# Create output directory if needed
if args.local_rank in [-1, 0]:
os.makedirs(args.output_dir, exist_ok=True)
logger.info("Saving model checkpoint to %s", args.output_dir)
# Save a trained model, configuration and tokenizer using `save_pretrained()`.
# They can then be reloaded using `from_pretrained()`
model_to_save = (model.module if hasattr(model, "module") else model
) # Take care of distributed/parallel training
model_to_save.save_pretrained(args.output_dir)
tokenizer.save_pretrained(args.output_dir)
# Good practice: save your training arguments together with the trained model
torch.save(args, os.path.join(args.output_dir, "training_args.bin"))
# Load a trained model and vocabulary that you have fine-tuned
if args.mlm:
model = AutoModelWithLMHead.from_pretrained(args.output_dir)
elif args.token_discrimination or args.mask_token_discrimination:
model = RobertaForTokenDiscrimination.from_pretrained(
args.output_dir)
else:
raise NotImplementedError(
'only mlm and token discrimination loss supported')
tokenizer = AutoTokenizer.from_pretrained(args.output_dir)
model.to(args.device)
# Evaluation
results = {}
if args.do_eval and args.local_rank in [-1, 0]:
checkpoints = [args.output_dir]
if args.eval_all_checkpoints:
checkpoints = list(
os.path.dirname(c) for c in sorted(
glob.glob(args.output_dir + "/**/" + WEIGHTS_NAME,
recursive=True)))
logging.getLogger("transformers.modeling_utils").setLevel(
logging.WARN) # Reduce logging
logger.info("Evaluate the following checkpoints: %s", checkpoints)
for checkpoint in checkpoints:
global_step = checkpoint.split(
"-")[-1] if len(checkpoints) > 1 else ""
prefix = checkpoint.split(
"/")[-1] if checkpoint.find("checkpoint") != -1 else ""
if args.mlm:
model = AutoModelWithLMHead.from_pretrained(checkpoint)
elif args.token_discrimination or args.mask_token_discrimination:
model = RobertaForTokenDiscrimination.from_pretrained(
checkpoint)
else:
raise NotImplementedError(
'only mlm and token discrimination loss supported')
model.to(args.device)
result = evaluate(args, model, tokenizer, prefix=prefix)
result = dict(
(k + "_{}".format(global_step), v) for k, v in result.items())
results.update(result)
return results
if not os.path.isdir(f".cache/{args.output_path}"):
dataset = load_dataset(f"data_scripts/{args.dataset_name}.py")
encoded = dataset.map(convert_to_features, batched=True)
encoded.save_to_disk(f".cache/{args.output_path}")
else:
encoded = load_from_disk(f".cache/{args.output_path}")
columns = [
'input_ids', 'attention_mask', 'labels', 'decoder_attention_mask'
]
encoded.set_format(type='torch', columns=columns)
device = 'cuda' if torch.cuda.is_available() else 'cpu'
if args.do_train:
if args.from_pretrained:
model = AutoModelWithLMHead.from_pretrained(args.model_select)
else:
config = AutoConfig.from_pretrained(args.model_select)
model = AutoModelWithLMHead.from_config(config)
train(args,
logger,
model,
tokenizer,
device,
encoded["train"],
collate_fn=collate_fn,
verbose=True)
def main():
parser = argparse.ArgumentParser()
# Required parameters
parser.add_argument("--train_data_file",
default=None,
type=str,
required=True,
help="The input training data file (a text file).")
parser.add_argument(
"--output_dir",
type=str,
required=True,
help=
"The output directory where the model predictions and checkpoints will be written.",
)
parser.add_argument(
"--data_dir",
type=str,
required=True,
help="The path to the input data directory.",
)
parser.add_argument(
"--model_type",
type=str,
required=True,
help="The model architecture to be trained or fine-tuned.",
)
# Other parameters
parser.add_argument(
"--eval_data_file",
default=None,
type=str,
help=
"An optional input evaluation data file to evaluate the perplexity on (a text file).",
)
parser.add_argument(
"--model_name_or_path",
default=None,
type=str,
help=
"The model checkpoint for weights initialization. Leave None if you want to train a model from scratch.",
)
parser.add_argument(
"--mlm",
action="store_true",
help=
"Train with masked-language modeling loss instead of language modeling."
)
parser.add_argument(
"--mlm_probability",
type=float,
default=0.15,
help="Ratio of tokens to mask for masked language modeling loss")
parser.add_argument(
"--config_name",
default=None,
type=str,
help=
"Optional pretrained config name or path if not the same as model_name_or_path. If both are None, initialize a new config.",
)
parser.add_argument(
"--tokenizer_name",
default=None,
type=str,
help=
"Optional pretrained tokenizer name or path if not the same as model_name_or_path. If both are None, initialize a new tokenizer.",
)
parser.add_argument(
"--cache_dir",
default=None,
type=str,
help=
"Optional directory to store the pre-trained models downloaded from s3 (instead of the default one)",
)
parser.add_argument(
"--block_size",
default=-1,
type=int,
help="Optional input sequence length after tokenization."
"The training dataset will be truncated in block of this size for training."
"Default to the model max input length for single sentence inputs (take into account special tokens).",
)
parser.add_argument("--do_train",
action="store_true",
help="Whether to run training.")
parser.add_argument("--do_eval",
action="store_true",
help="Whether to run eval on the dev set.")
parser.add_argument(
"--evaluate_during_training",
action="store_true",
help="Run evaluation during training at each logging step.")
parser.add_argument("--per_gpu_train_batch_size",
default=4,
type=int,
help="Batch size per GPU/CPU for training.")
parser.add_argument("--per_gpu_eval_batch_size",
default=4,
type=int,
help="Batch size per GPU/CPU for evaluation.")
parser.add_argument(
"--gradient_accumulation_steps",
type=int,
default=1,
help=
"Number of updates steps to accumulate before performing a backward/update pass.",
)
parser.add_argument("--local_rank",
type=int,
default=-1,
help="For distributed training: local_rank")
parser.add_argument("--learning_rate",
default=5e-5,
type=float,
help="The initial learning rate for Adam.")
parser.add_argument("--weight_decay",
default=0.0,
type=float,
help="Weight decay if we apply some.")
parser.add_argument("--adam_epsilon",
default=1e-8,
type=float,
help="Epsilon for Adam optimizer.")
parser.add_argument("--max_grad_norm",
default=1.0,
type=float,
help="Max gradient norm.")
parser.add_argument("--num_train_epochs",
default=1.0,
type=float,
help="Total number of training epochs to perform.")
parser.add_argument(
"--max_steps",
default=-1,
type=int,
help=
"If > 0: set total number of training steps to perform. Override num_train_epochs.",
)
parser.add_argument("--warmup_steps",
default=0,
type=int,
help="Linear warmup over warmup_steps.")
parser.add_argument("--logging_steps",
type=int,
default=500,
help="Log every X updates steps.")
parser.add_argument("--save_steps",
type=int,
default=500,
help="Save checkpoint every X updates steps.")
parser.add_argument(
"--save_total_limit",
type=int,
default=None,
help=
"Limit the total amount of checkpoints, delete the older checkpoints in the output_dir, does not delete by default",
)
parser.add_argument(
"--eval_all_checkpoints",
action="store_true",
help=
"Evaluate all checkpoints starting with the same prefix as model_name_or_path ending and ending with step number",
)
parser.add_argument("--no_cuda",
action="store_true",
help="Avoid using CUDA when available")
parser.add_argument("--overwrite_output_dir",
action="store_true",
help="Overwrite the content of the output directory")
parser.add_argument(
"--overwrite_cache",
action="store_true",
help="Overwrite the cached training and evaluation sets")
parser.add_argument("--seed",
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
"--fp16",
action="store_true",
help=
"Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit",
)
parser.add_argument(
"--fp16_opt_level",
type=str,
default="O1",
help=
"For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
"See details at https://nvidia.github.io/apex/amp.html",
)
parser.add_argument(
"--temperature",
type=float,
default=1.0,
help=
"temperature of 1.0 has no effect, lower tend toward greedy sampling",
)
parser.add_argument("--num_return_sequences",
type=int,
default=1,
help="The number of samples to generate.")
parser.add_argument(
"--repetition_penalty",
type=float,
default=1.0,
help="primarily useful for CTRL model; in that case, use 1.2")
parser.add_argument("--length", type=int, default=20)
parser.add_argument("--k", type=int, default=0)
parser.add_argument("--p", type=float, default=0.9)
parser.add_argument("--stop_token",
type=str,
default=None,
help="Token at which text generation is stopped")
parser.add_argument("--sep_token",
type=str,
default="<sep>",
help="Token to separate journal and reflection")
args = parser.parse_args()
if args.model_type in ["bert", "roberta", "distilbert", "camembert"
] and not args.mlm:
raise ValueError(
"BERT and RoBERTa-like models do not have LM heads but masked LM heads. They must be run using the --mlm "
"flag (masked language modeling).")
if args.eval_data_file is None and args.do_eval:
raise ValueError(
"Cannot do evaluation without an evaluation data file. Either supply a file to --eval_data_file "
"or remove the --do_eval argument.")
if (os.path.exists(args.output_dir) and os.listdir(args.output_dir)
and args.do_train and not args.overwrite_output_dir
and not args.should_continue):
raise ValueError(
"Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome."
.format(args.output_dir))
# Setup CUDA, GPU & distributed training
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
args.n_gpu = 0 if args.no_cuda else torch.cuda.device_count()
else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
torch.distributed.init_process_group(backend="nccl")
args.n_gpu = 1
args.device = device
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN,
)
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
args.local_rank,
device,
args.n_gpu,
bool(args.local_rank != -1),
args.fp16,
)
# Set seed
set_seed(args)
# Load pretrained model and tokenizer
if args.local_rank not in [-1, 0]:
torch.distributed.barrier(
) # Barrier to make sure only the first process in distributed training download model & vocab
if args.config_name:
config = AutoConfig.from_pretrained(args.config_name,
cache_dir=args.cache_dir)
elif args.model_name_or_path:
config = AutoConfig.from_pretrained(args.model_name_or_path,
cache_dir=args.cache_dir)
else:
# When we release a pip version exposing CONFIG_MAPPING,
# we can do `config = CONFIG_MAPPING[args.model_type]()`.
raise ValueError(
"You are instantiating a new config instance from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --config_name")
if args.tokenizer_name:
tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_name,
cache_dir=args.cache_dir)
elif args.model_name_or_path:
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path,
cache_dir=args.cache_dir)
else:
raise ValueError(
"You are instantiating a new tokenizer from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --tokenizer_name")
if args.block_size <= 0:
args.block_size = tokenizer.max_len
# Our input block size will be the max possible for the model
else:
args.block_size = min(args.block_size, tokenizer.max_len)
if args.model_name_or_path:
model = AutoModelWithLMHead.from_pretrained(
args.model_name_or_path,
from_tf=bool(".ckpt" in args.model_name_or_path),
config=config,
cache_dir=args.cache_dir,
)
else:
logger.info("Training new model from scratch")
model = AutoModelWithLMHead.from_config(config)
model.to(args.device)
if args.local_rank == 0:
torch.distributed.barrier(
) # End of barrier to make sure only the first process in distributed training download model & vocab
logger.info("Training/evaluation parameters %s", args)
# Training
if args.do_train:
if args.local_rank not in [-1, 0]:
torch.distributed.barrier(
) # Barrier to make sure only the first process in distributed training process the dataset, and the others will use the cache
train_dataset = load_and_cache_examples(args,
tokenizer,
evaluate=False)
if args.local_rank == 0:
torch.distributed.barrier()
global_step, tr_loss = train(args, train_dataset, model, tokenizer)
logger.info(" global_step = %s, average loss = %s", global_step,
tr_loss)
def main():
# See all possible arguments in src/transformers/training_args.py
# or by passing the --help flag to this script.
# We now keep distinct sets of args, for a cleaner separation of concerns.
parser = HfArgumentParser(
(ModelArguments, DataTrainingArguments, TrainingArguments))
model_args, data_args, training_args = parser.parse_args_into_dataclasses()
if data_args.eval_data_file is None and training_args.do_eval:
raise ValueError(
"Cannot do evaluation without an evaluation data file. Either supply a file to --eval_data_file "
"or remove the --do_eval argument.")
if (os.path.exists(training_args.output_dir)
and os.listdir(training_args.output_dir) and training_args.do_train
and not training_args.overwrite_output_dir):
raise ValueError(
f"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
)
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO
if training_args.local_rank in [-1, 0] else logging.WARN,
)
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
training_args.local_rank,
training_args.device,
training_args.n_gpu,
bool(training_args.local_rank != -1),
training_args.fp16,
)
logger.info("Training/evaluation parameters %s", training_args)
# Set seed
set_seed(training_args.seed)
# Load pretrained model and tokenizer
#
# Distributed training:
# The .from_pretrained methods guarantee that only one local process can concurrently
# download model & vocab.
if model_args.config_name:
config = AutoConfig.from_pretrained(model_args.config_name,
cache_dir=model_args.cache_dir)
elif model_args.model_name_or_path:
config = AutoConfig.from_pretrained(model_args.model_name_or_path,
cache_dir=model_args.cache_dir)
else:
config = CONFIG_MAPPING[model_args.model_type]()
logger.warning(
"You are instantiating a new config instance from scratch.")
with Path('foodbert/data/used_ingredients.json').open() as f:
used_ingredients = json.load(f) # Dont seperate these
tokenizer = BertTokenizer(
vocab_file='foodbert/data/bert-base-cased-vocab.txt',
do_lower_case=False,
max_len=128,
never_split=used_ingredients
) # For one sentence instruction, longer shouldn't be necessary
if model_args.model_name_or_path:
model = AutoModelWithLMHead.from_pretrained(
model_args.model_name_or_path,
from_tf=bool(".ckpt" in model_args.model_name_or_path),
config=config,
cache_dir=model_args.cache_dir,
)
else:
logger.info("Training new model from scratch")
model = AutoModelWithLMHead.from_config(config)
model.resize_token_embeddings(len(tokenizer))
if config.model_type in ["bert", "roberta", "distilbert", "camembert"
] and not data_args.mlm:
raise ValueError(
"BERT and RoBERTa-like models do not have LM heads but masked LM heads. They must be run using the --mlm "
"flag (masked language modeling).")
if data_args.block_size <= 0:
data_args.block_size = tokenizer.max_len
# Our input block size will be the max possible for the model
else:
data_args.block_size = min(data_args.block_size, tokenizer.max_len)
# Get datasets
train_dataset = (get_dataset(
data_args, tokenizer=tokenizer, local_rank=training_args.local_rank)
if training_args.do_train else None)
eval_dataset = (get_dataset(data_args,
tokenizer=tokenizer,
local_rank=training_args.local_rank,
evaluate=True)
if training_args.do_eval else None)
data_collator = DataCollatorForLanguageModeling(
tokenizer=tokenizer,
mlm=data_args.mlm,
mlm_probability=data_args.mlm_probability)
# Make sure checkpoint recovery and continous training works on GPU, probably we need to make sure to push all parameters to the gpu
# Solves bug in Trainer https://github.com/huggingface/transformers/issues/4240
model.to(training_args.device)
# Initialize our Trainer
trainer = Trainer(
model=model,
args=training_args,
data_collator=data_collator,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
prediction_loss_only=True,
)
# Training
if training_args.do_train:
model_path = (model_args.model_name_or_path
if model_args.model_name_or_path is not None
and os.path.isdir(model_args.model_name_or_path) else
None)
trainer.train(model_path=model_path)
trainer.save_model()
# For convenience, we also re-save the tokenizer to the same directory,
# so that you can share your model easily on huggingface.co/models =)
if trainer.is_world_master():
tokenizer.save_pretrained(training_args.output_dir)
# Evaluation
results = {}
if training_args.do_eval and training_args.local_rank in [-1, 0]:
logger.info("*** Evaluate ***")
eval_output = trainer.evaluate()
perplexity = math.exp(eval_output["eval_loss"])
result = {"perplexity": perplexity}
output_eval_file = os.path.join(training_args.output_dir,
"eval_results_lm.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
results.update(result)
return results
def main():
# See all possible arguments in src/transformers/training_args.py
# or by passing the --help flag to this script.
# We now keep distinct sets of args, for a cleaner separation of concerns.
parser = HfArgumentParser(
(ModelArguments, DataTrainingArguments, TrainingArguments))
model_args, data_args, training_args = parser.parse_args_into_dataclasses()
if training_args.do_train:
raise ValueError("Training not supported with this script.")
if data_args.eval_data_file is None:
raise ValueError(
"Cannot do evaluation without an evaluation data file.")
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO
if training_args.local_rank in [-1, 0] else logging.WARN,
)
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
training_args.local_rank,
training_args.device,
training_args.n_gpu,
bool(training_args.local_rank != -1),
training_args.fp16,
)
logger.info("Training/evaluation parameters %s", training_args)
# Set seed
set_seed(training_args.seed)
# Load pretrained model and tokenizer
#
# Distributed training:
# The .from_pretrained methods guarantee that only one local process can concurrently
# download model & vocab.
if model_args.config_name:
config = AutoConfig.from_pretrained(model_args.config_name,
cache_dir=model_args.cache_dir)
elif model_args.model_name_or_path:
config = AutoConfig.from_pretrained(model_args.model_name_or_path,
cache_dir=model_args.cache_dir)
else:
config = CONFIG_MAPPING[model_args.model_type]()
logger.warning(
"You are instantiating a new config instance from scratch.")
if model_args.tokenizer_name:
tokenizer = AutoTokenizer.from_pretrained(
model_args.tokenizer_name, cache_dir=model_args.cache_dir)
elif model_args.model_name_or_path:
tokenizer = AutoTokenizer.from_pretrained(
model_args.model_name_or_path, cache_dir=model_args.cache_dir)
else:
raise ValueError(
"You are instantiating a new tokenizer from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --tokenizer_name")
inline_meta = None
if model_args.controllable_model == 'ctrl-gpt2':
print("Loaded added tokens:", tokenizer.added_tokens_encoder)
inline_meta = TARGET_CTRL_CODE
assert inline_meta in tokenizer.get_vocab().keys()
print("Using inline meta:", inline_meta)
if model_args.model_name_or_path:
controllable_model = model_args.controllable_model
if controllable_model and controllable_model.startswith('affect-gpt2'):
logger.info("Using AffectLM for evaluation")
model = AffectGPT2LMHeadModel.from_pretrained(
model_args.model_name_or_path,
from_tf=bool(".ckpt" in model_args.model_name_or_path),
config=config,
cache_dir=model_args.cache_dir,
)
# Evaluate with non-toxic affect
beta = int(
re.search(r'beta-([0-9]+)', controllable_model).group(1))
logging.info(f"Setting AffectLM beta to {beta}")
model.affect.beta = beta
affect_label = F.one_hot(torch.LongTensor([AFFECT_LABEL_IDX]),
num_classes=AFFECT_NUM_LABELS).float().to(
training_args.device)
model.set_affect_labels(affect_label)
else:
model = AutoModelWithLMHead.from_pretrained(
model_args.model_name_or_path,
from_tf=bool(".ckpt" in model_args.model_name_or_path),
config=config,
cache_dir=model_args.cache_dir,
)
else:
logger.info("Training new model from scratch")
model = AutoModelWithLMHead.from_config(config)
model.resize_token_embeddings(len(tokenizer))
if config.model_type in ["bert", "roberta", "distilbert", "camembert"
] and not data_args.mlm:
raise ValueError(
"BERT and RoBERTa-like models do not have LM heads but masked LM heads. They must be run using the --mlm "
"flag (masked language modeling).")
if data_args.block_size <= 0:
data_args.block_size = tokenizer.max_len
# Our input block size will be the max possible for the model
else:
data_args.block_size = min(data_args.block_size, tokenizer.max_len)
# Get datasets
eval_dataset = get_dataset(data_args,
tokenizer=tokenizer,
local_rank=training_args.local_rank,
inline_meta=inline_meta)
data_collator = DataCollatorForLanguageModeling(
tokenizer=tokenizer,
mlm=data_args.mlm,
mlm_probability=data_args.mlm_probability)
# Initialize our Trainer
trainer = Trainer(
model=model,
args=training_args,
data_collator=data_collator,
eval_dataset=eval_dataset,
prediction_loss_only=True,
)
# Evaluation
results = {}
if training_args.local_rank in [-1, 0]:
logger.info("*** Evaluate ***")
eval_output = trainer.evaluate()
perplexity = math.exp(eval_output["loss"])
result = {"perplexity": perplexity}
output_eval_file = os.path.join(training_args.output_dir,
"eval_results_lm.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
results.update(result)
return results
def main():
# See all possible arguments in src/transformers/training_args.py
# or by passing the --help flag to this script.
# We now keep distinct sets of args, for a cleaner separation of concerns.
parser = HfArgumentParser(
(ModelArguments, DataTrainingArguments, TrainingArguments))
model_args, data_args, training_args = parser.parse_args_into_dataclasses()
if data_args.eval_data_file is None and training_args.do_eval:
raise ValueError(
"Cannot do evaluation without an evaluation data file. Either supply a file to --eval_data_file "
"or remove the --do_eval argument.")
if (os.path.exists(training_args.output_dir)
and os.listdir(training_args.output_dir) and training_args.do_train
and not training_args.overwrite_output_dir):
raise ValueError(
f"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
)
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO
if training_args.local_rank in [-1, 0] else logging.WARN,
)
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
training_args.local_rank,
training_args.device,
training_args.n_gpu,
bool(training_args.local_rank != -1),
training_args.fp16,
)
logger.info("Training/evaluation parameters %s", training_args)
# Set seed
set_seed(training_args.seed)
# Load pretrained model and tokenizer
#
# Distributed training:
# The .from_pretrained methods guarantee that only one local process can concurrently
# download model & vocab.
if model_args.config_name:
config = AutoConfig.from_pretrained(model_args.config_name,
cache_dir=model_args.cache_dir)
elif model_args.model_name_or_path:
config = AutoConfig.from_pretrained(model_args.model_name_or_path,
cache_dir=model_args.cache_dir)
else:
config = CONFIG_MAPPING[model_args.model_type]()
logger.warning(
"You are instantiating a new config instance from scratch.")
if model_args.tokenizer_name:
tokenizer = AutoTokenizer.from_pretrained(
model_args.tokenizer_name, cache_dir=model_args.cache_dir)
elif model_args.model_name_or_path:
tokenizer = AutoTokenizer.from_pretrained(
model_args.model_name_or_path, cache_dir=model_args.cache_dir)
else:
raise ValueError(
"You are instantiating a new tokenizer from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --tokenizer_name")
if model_args.model_name_or_path:
model = AutoModelWithLMHead.from_pretrained(
model_args.model_name_or_path,
from_tf=bool(".ckpt" in model_args.model_name_or_path),
config=config,
cache_dir=model_args.cache_dir,
)
else:
logger.info("Training new model from scratch")
model = AutoModelWithLMHead.from_config(config)
model.resize_token_embeddings(len(tokenizer))
if config.model_type in ["bert", "roberta", "distilbert", "camembert"
] and not data_args.mlm:
raise ValueError(
"BERT and RoBERTa-like models do not have LM heads but masked LM heads. They must be run using the --mlm "
"flag (masked language modeling).")
if data_args.block_size <= 0:
data_args.block_size = tokenizer.max_len
# Our input block size will be the max possible for the model
else:
data_args.block_size = min(data_args.block_size, tokenizer.max_len)
# Get datasets
train_dataset = (get_dataset(
data_args, tokenizer=tokenizer, local_rank=training_args.local_rank)
if training_args.do_train else None)
eval_dataset = (get_dataset(data_args,
tokenizer=tokenizer,
local_rank=training_args.local_rank,
evaluate=True)
if training_args.do_eval else None)
data_collator = DataCollatorForLanguageModeling(
tokenizer=tokenizer,
mlm=data_args.mlm,
mlm_probability=data_args.mlm_probability)
# Initialize our Trainer
trainer = Trainer(
model=model,
args=training_args,
data_collator=data_collator,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
prediction_loss_only=True,
)
# Training
if training_args.do_train:
model_path = (model_args.model_name_or_path
if model_args.model_name_or_path is not None
and os.path.isdir(model_args.model_name_or_path) else
None)
trainer.train(model_path=model_path)
trainer.save_model()
# Evaluation
results = {}
if training_args.do_eval and training_args.local_rank in [-1, 0]:
logger.info("*** Evaluate ***")
eval_output = trainer.evaluate()
perplexity = math.exp(eval_output["loss"])
result = {"perplexity": perplexity}
output_eval_file = os.path.join(training_args.output_dir,
"eval_results_lm.txt")
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
results.update(result)
return results
def main():
# See all possible arguments in src/transformers/training_args.py
# or by passing the --help flag to this script.
# We now keep distinct sets of args, for a cleaner separation of concerns.
parser = HfArgumentParser(
(ModelArguments, DataTrainingArguments, TrainingArguments))
model_args, data_args, training_args = parser.parse_args_into_dataclasses()
print('init')
if (os.path.exists(training_args.output_dir)
and os.listdir(training_args.output_dir) and training_args.do_train
and not training_args.overwrite_output_dir):
raise ValueError(
f"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
)
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO
if training_args.local_rank in [-1, 0] else logging.WARN,
)
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
training_args.local_rank,
training_args.device,
training_args.n_gpu,
bool(training_args.local_rank != -1),
training_args.fp16,
)
logger.info("Training/evaluation parameters %s", training_args)
# Set seed
set_seed(training_args.seed)
# Load pretrained model and tokenizer
#
# Distributed training:
# The .from_pretrained methods guarantee that only one local process can concurrently
# download model & vocab.
if model_args.config_name:
config = AutoConfig.from_pretrained(model_args.config_name,
cache_dir=model_args.cache_dir)
elif model_args.model_name_or_path:
config = AutoConfig.from_pretrained(model_args.model_name_or_path,
cache_dir=model_args.cache_dir)
else:
config = CONFIG_MAPPING[model_args.model_type]()
logger.warning(
"You are instantiating a new config instance from scratch.")
if model_args.model_name_or_path:
model = AutoModelWithLMHead.from_pretrained(
model_args.model_name_or_path,
from_tf=bool(".ckpt" in model_args.model_name_or_path),
config=config,
cache_dir=model_args.cache_dir,
)
else:
logger.info("Training new model from scratch")
model = AutoModelWithLMHead.from_config(config)
model.resize_token_embeddings(config.vocab_size)
if data_args.block_size <= 0:
data_args.block_size = config.n_positions
# Our input block size will be the max possible for the model
else:
data_args.block_size = min(data_args.block_size, config.n_positions)
print('get data')
# Get datasets
dataset = TokensDataset(f'{data_args.data_dir}/data.npy',
f'{data_args.data_dir}/index.npy',
mmap=False,
subset_size=1.0)
train_dataset = dataset.train(data_args.block_size)
eval_dataset = dataset.valid(data_args.block_size)
print('start')
# Initialize our Trainer
trainer = MyTrainer(
model=model,
args=training_args,
data_collator=prepare_batch,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
)
# Training
if training_args.do_train:
model_path = (model_args.model_name_or_path
if model_args.model_name_or_path is not None
and os.path.isdir(model_args.model_name_or_path) else
None)
trainer.train(model_path=model_path)
trainer.save_model()
# Evaluation
results = {}
if training_args.do_eval:
logger.info("*** Evaluate ***")
eval_output = trainer.evaluate()
perplexity = math.exp(eval_output["eval_loss"])
result = {"perplexity": perplexity}
output_eval_file = os.path.join(training_args.output_dir,
"eval_results_lm.txt")
if trainer.is_world_master():
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
results.update(result)
return results
def main():
# See all possible arguments in src/transformers/training_args.py
# or by passing the --help flag to this script.
# We now keep distinct sets of args, for a cleaner separation of concerns.
parser = HfArgumentParser((ModelArguments, DataTrainingArguments,
TrainingArguments, AdapterArguments))
model_args, data_args, training_args, adapter_args = parser.parse_args_into_dataclasses(
)
if data_args.eval_data_file is None and training_args.do_eval:
raise ValueError(
"Cannot do evaluation without an evaluation data file. Either supply a file to --eval_data_file "
"or remove the --do_eval argument.")
if (os.path.exists(training_args.output_dir)
and os.listdir(training_args.output_dir) and training_args.do_train
and not training_args.overwrite_output_dir):
raise ValueError(
f"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
)
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO
if training_args.local_rank in [-1, 0] else logging.WARN,
)
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
training_args.local_rank,
training_args.device,
training_args.n_gpu,
bool(training_args.local_rank != -1),
training_args.fp16,
)
logger.info("Training/evaluation parameters %s", training_args)
# Set seed
set_seed(training_args.seed)
# Load pretrained model and tokenizer
#
# Distributed training:
# The .from_pretrained methods guarantee that only one local process can concurrently
# download model & vocab.
if model_args.config_name:
config = AutoConfig.from_pretrained(model_args.config_name,
cache_dir=model_args.cache_dir)
elif model_args.model_name_or_path:
config = AutoConfig.from_pretrained(model_args.model_name_or_path,
cache_dir=model_args.cache_dir)
else:
config = CONFIG_MAPPING[model_args.model_type]()
logger.warning(
"You are instantiating a new config instance from scratch.")
if model_args.tokenizer_name:
tokenizer = AutoTokenizer.from_pretrained(
model_args.tokenizer_name, cache_dir=model_args.cache_dir)
elif model_args.model_name_or_path:
tokenizer = AutoTokenizer.from_pretrained(
model_args.model_name_or_path, cache_dir=model_args.cache_dir)
else:
raise ValueError(
"You are instantiating a new tokenizer from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --tokenizer_name")
if model_args.model_name_or_path:
model = AutoModelWithLMHead.from_pretrained(
model_args.model_name_or_path,
from_tf=bool(".ckpt" in model_args.model_name_or_path),
config=config,
cache_dir=model_args.cache_dir,
)
else:
logger.info("Training new model from scratch")
model = AutoModelWithLMHead.from_config(config)
model.resize_token_embeddings(len(tokenizer))
# Setup adapters
if adapter_args.train_adapter:
language = adapter_args.language
if not language:
raise ValueError(
"--language flag must be set when training an adapter")
# check if language adapter already exists, otherwise add it
if language not in model.config.adapters.adapter_list(
AdapterType.text_lang):
# resolve the adapter config
adapter_config = AdapterConfig.load(
adapter_args.adapter_config,
non_linearity=adapter_args.adapter_non_linearity,
reduction_factor=adapter_args.adapter_reduction_factor,
)
# load a pre-trained from Hub if specified
if adapter_args.load_adapter:
model.load_adapter(
adapter_args.load_adapter,
AdapterType.text_lang,
config=adapter_config,
load_as=language,
)
# otherwise, add a fresh adapter
else:
model.add_adapter(language,
AdapterType.text_lang,
config=adapter_config)
# Freeze all model weights except of those of this adapter & use this adapter in every forward pass
model.train_adapter([language])
if config.model_type in ["bert", "roberta", "distilbert", "camembert"
] and not data_args.mlm:
raise ValueError(
"BERT and RoBERTa-like models do not have LM heads but masked LM heads. They must be run using the"
"--mlm flag (masked language modeling).")
if data_args.block_size <= 0:
data_args.block_size = tokenizer.max_len
# Our input block size will be the max possible for the model
else:
data_args.block_size = min(data_args.block_size, tokenizer.max_len)
# Get datasets
train_dataset = (get_dataset(
data_args, tokenizer=tokenizer, cache_dir=model_args.cache_dir)
if training_args.do_train else None)
eval_dataset = (get_dataset(data_args,
tokenizer=tokenizer,
evaluate=True,
cache_dir=model_args.cache_dir)
if training_args.do_eval else None)
if config.model_type == "xlnet":
data_collator = DataCollatorForPermutationLanguageModeling(
tokenizer=tokenizer,
plm_probability=data_args.plm_probability,
max_span_length=data_args.max_span_length,
)
else:
data_collator = DataCollatorForLanguageModeling(
tokenizer=tokenizer,
mlm=data_args.mlm,
mlm_probability=data_args.mlm_probability)
# Initialize our Trainer
trainer = Trainer(
model=model,
args=training_args,
data_collator=data_collator,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
prediction_loss_only=True,
do_save_full_model=not adapter_args.train_adapter,
do_save_adapters=adapter_args.train_adapter,
)
# Training
if training_args.do_train:
model_path = (model_args.model_name_or_path
if model_args.model_name_or_path is not None
and os.path.isdir(model_args.model_name_or_path) else
None)
trainer.train(model_path=model_path)
trainer.save_model()
# For convenience, we also re-save the tokenizer to the same directory,
# so that you can share your model easily on huggingface.co/models =)
if trainer.is_world_master():
tokenizer.save_pretrained(training_args.output_dir)
# Evaluation
results = {}
if training_args.do_eval:
logger.info("*** Evaluate ***")
eval_output = trainer.evaluate()
perplexity = math.exp(eval_output["eval_loss"])
result = {"perplexity": perplexity}
output_eval_file = os.path.join(training_args.output_dir,
"eval_results_lm.txt")
if trainer.is_world_master():
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
results.update(result)
return results
def main():
# See all possible arguments in src/transformers/training_args.py
# or by passing the --help flag to this script.
# We now keep distinct sets of args, for a cleaner separation of concerns.
model_args = ModelArguments(model_name_or_path="gpt2",
model_type="gpt2",
cache_dir="/u0/psvadrev/.cache")
data_args = DataTrainingArguments(
train_data_file="6_genre_clean_training_data.txt",
eval_data_file="6_genre_eval_data.txt",
line_by_line=True,
mlm=False,
block_size=512,
overwrite_cache=True,
)
training_args = TrainingArguments(
output_dir="story_generator_6_genre_eval",
overwrite_output_dir=True,
do_train=True,
do_eval=True,
do_predict=False,
evaluate_during_training=False,
logging_steps=500,
per_device_train_batch_size=4,
num_train_epochs=10,
save_total_limit=1,
save_steps=1000000,
)
if data_args.eval_data_file is None and training_args.do_eval:
raise ValueError(
"Cannot do evaluation without an evaluation data file. Either supply a file to --eval_data_file "
"or remove the --do_eval argument.")
if (os.path.exists(training_args.output_dir)
and os.listdir(training_args.output_dir) and training_args.do_train
and not training_args.overwrite_output_dir):
raise ValueError(
f"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
)
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO
if training_args.local_rank in [-1, 0] else logging.WARN,
)
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
training_args.local_rank,
training_args.device,
training_args.n_gpu,
bool(training_args.local_rank != -1),
training_args.fp16,
)
logger.info("Training/evaluation parameters %s", training_args)
# Set seed
set_seed(training_args.seed)
# Load pretrained model and tokenizer
#
# Distributed training:
# The .from_pretrained methods guarantee that only one local process can concurrently
# download model & vocab.
if model_args.config_name:
config = AutoConfig.from_pretrained(model_args.config_name,
cache_dir=model_args.cache_dir)
elif model_args.model_name_or_path:
config = AutoConfig.from_pretrained(model_args.model_name_or_path,
cache_dir=model_args.cache_dir)
else:
config = CONFIG_MAPPING[model_args.model_type]()
logger.warning(
"You are instantiating a new config instance from scratch.")
if model_args.tokenizer_name:
tokenizer = AutoTokenizer.from_pretrained(
model_args.tokenizer_name, cache_dir=model_args.cache_dir)
elif model_args.model_name_or_path:
tokenizer = AutoTokenizer.from_pretrained(
model_args.model_name_or_path, cache_dir=model_args.cache_dir)
else:
raise ValueError(
"You are instantiating a new tokenizer from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --tokenizer_name")
if model_args.model_name_or_path:
model = AutoModelWithLMHead.from_pretrained(
model_args.model_name_or_path,
from_tf=bool(".ckpt" in model_args.model_name_or_path),
config=config,
cache_dir=model_args.cache_dir,
)
else:
logger.info("Training new model from scratch")
model = AutoModelWithLMHead.from_config(config)
special_tokens_dict = {
"bos_token":
"<BOS>",
"eos_token":
"<EOS>",
"pad_token":
"<PAD>",
"additional_special_tokens": [
"<superhero>",
"<action>",
"<drama>",
"<thriller>",
"<horror>",
"<sci_fi>",
],
}
num_added_toks = tokenizer.add_special_tokens(special_tokens_dict)
model.resize_token_embeddings(len(tokenizer))
if data_args.block_size <= 0:
data_args.block_size = tokenizer.max_len
# Our input block size will be the max possible for the model
else:
data_args.block_size = min(data_args.block_size, tokenizer.max_len)
# Get datasets
train_dataset = (get_dataset(data_args, tokenizer=tokenizer)
if training_args.do_train else None)
eval_dataset = (get_dataset(data_args, tokenizer=tokenizer, evaluate=True)
if training_args.do_eval else None)
data_collator = DataCollatorForLanguageModeling(
tokenizer=tokenizer,
mlm=data_args.mlm,
mlm_probability=data_args.mlm_probability,
)
# Initialize our Trainer
trainer = Trainer(
model=model,
args=training_args,
data_collator=data_collator,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
prediction_loss_only=True,
)
# Training
if training_args.do_train:
model_path = (model_args.model_name_or_path
if model_args.model_name_or_path is not None
and os.path.isdir(model_args.model_name_or_path) else
None)
trainer.train(model_path=model_path)
trainer.save_model()
# For convenience, we also re-save the tokenizer to the same directory,
# so that you can share your model easily on huggingface.co/models =)
if trainer.is_world_master():
tokenizer.save_pretrained(training_args.output_dir)
# Evaluation
results = {}
if training_args.do_eval:
logger.info("*** Evaluate ***")
eval_output = trainer.evaluate()
perplexity = math.exp(eval_output["eval_loss"])
result = {"perplexity": perplexity}
output_eval_file = os.path.join(training_args.output_dir,
"eval_results_lm.txt")
if trainer.is_world_master():
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
results.update(result)
return results
def __init__(
self,
model: str = None,
config: Union[str, GPT2Config] = None,
vocab_file: str = None,
merges_file: str = None,
cache_dir: str = "aitextgen",
tf_gpt2: str = None,
to_gpu: bool = False,
to_fp16: bool = False,
verbose: bool = False,
torchscript: bool = False,
ts_to_trace: bool = False,
bos_token: str = None,
eos_token: str = None,
unk_token: str = None,
**kwargs,
) -> None:
if not verbose:
for module in [
"transformers.file_utils",
"transformers.configuration_utils",
"transformers.tokenization_utils",
"filelock",
"transformers.modeling_gpt2",
]:
logging.getLogger(module).setLevel(logging.WARN)
logging.getLogger("transformers.modeling_utils").setLevel(
logging.ERROR)
if torchscript:
assert model
logger.info(f"Loading traced GPT-2 model from provided {model}.")
if config is None:
config = GPT2Config()
self.torchscript = True
self.model = GPT2LMHeadModel(config)
# Transpose the traced model attributes to a GPT2LMHeadModel class
# so it can inherit its functions
pt_model = torch.jit.load(model)
self.model.transformer = pt_model.transformer
self.model.lm_head = pt_model.lm_head
elif tf_gpt2:
# Download + convert the TF weights if a PyTorch model has not been created
if not os.path.isfile(
os.path.join(cache_dir, f"pytorch_model_{tf_gpt2}.bin")):
assert tf_gpt2 in [
"124M",
"355M",
"774M",
"1558M",
], "Invalid TensorFlow GPT-2 model size."
logger.info(
f"Downloading the {tf_gpt2} GPT-2 TensorFlow weights/config "
+ "from Google's servers")
download_gpt2(cache_dir, tf_gpt2)
logger.info(
f"Converting the {tf_gpt2} GPT-2 TensorFlow weights to PyTorch."
)
config_path = os.path.join(cache_dir, tf_gpt2, "hparams.json")
convert_gpt2_checkpoint_to_pytorch(
os.path.join(cache_dir, tf_gpt2),
config_path,
cache_dir,
)
os.rename(
os.path.join(cache_dir, f"pytorch_model.bin"),
os.path.join(cache_dir, f"pytorch_model_{tf_gpt2}.bin"),
)
os.rename(
os.path.join(cache_dir, f"config.json"),
os.path.join(cache_dir, f"config_{tf_gpt2}.json"),
)
logger.info(f"Loading {tf_gpt2} GPT-2 model from /{cache_dir}.")
model = os.path.join(cache_dir, f"pytorch_model_{tf_gpt2}.bin")
config = os.path.join(cache_dir, f"config_{tf_gpt2}.json")
self.model = GPT2LMHeadModel.from_pretrained(model, config=config)
elif model and os.path.exists(model):
# A pytorch_model.bin (+ optional config/config.json) is provided
logger.info(f"Loading GPT-2 model from provided {model}.")
if config is None:
config = GPT2Config()
if ts_to_trace:
config.torchscript = True
self.model = GPT2LMHeadModel.from_pretrained(model, config=config)
elif config:
if ts_to_trace:
config.torchscript = True
# Manually construct a GPT-2 model from scratch
logger.info("Constructing GPT-2 model from provided config.")
self.model = AutoModelWithLMHead.from_config(config=config)
else:
# Download and cache model from Huggingface
if os.path.isdir(cache_dir) and len(os.listdir(cache_dir)) > 0:
logger.info(
f"Loading {model or 'gpt2'} model from /{cache_dir}.")
else:
logger.info(
f"Downloading {model or 'gpt2'} model to /{cache_dir}.")
self.model = GPT2LMHeadModel.from_pretrained(
model or "gpt2", cache_dir=cache_dir, torchscript=ts_to_trace)
if model and "gpt2" not in model:
logger.info(f"Using the tokenizer for {model}.")
self.tokenizer = GPT2Tokenizer.from_pretrained(
model,
cache_dir=cache_dir,
)
if self.tokenizer is None:
# Update tokenizer settings (if not set already)
args = locals()
custom_tokenizer = False
for attr in [
"vocab_file",
"merges_file",
"bos_token",
"eos_token",
"unk_token",
]:
if args[attr] is not None:
custom_tokenizer = True
setattr(self, attr, args[attr])
if custom_tokenizer:
logger.info("Using a custom tokenizer.")
else:
logger.info("Using the default GPT-2 Tokenizer.")
self.tokenizer = GPT2Tokenizer(
vocab_file=self.vocab_file,
merges_file=self.merges_file,
bos_token=self.bos_token,
eos_token=self.eos_token,
unk_token=self.unk_token,
pad_token=self.pad_token,
)
if to_gpu:
if to_fp16:
self.to_fp16()
self.to_gpu()
def train(args):
wandb.init(project="lm-finetuning", config=args, tags=args.tags)
if args.save_dir == None:
args.save_dir = wandb.run.dir
tokenizer = AutoTokenizer.from_pretrained(args.tokenizer, use_fast=True)
if args.from_scratch:
config = AutoConfig.from_pretrained(args.config)
model = AutoModelWithLMHead.from_config(config=config).to(args.device)
else:
model = AutoModelWithLMHead.from_pretrained(args.checkpoint,
from_tf=args.from_tf).to(
args.device)
tokenizer.add_special_tokens(
{'additional_special_tokens': args.control_codes})
model.resize_token_embeddings(len(tokenizer))
print(f"Params: {n_params(model)}")
train_dataset = TextDataset(args.train_path, tokenizer, args)
val_dataset = TextDataset(args.val_path, tokenizer, args)
def collate(examples):
if tokenizer._pad_token is None:
return pad_sequence(examples, batch_first=True)
return pad_sequence(examples,
batch_first=True,
padding_value=tokenizer.pad_token_id)
train_dataloader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=4,
collate_fn=collate)
val_dataloader = torch.utils.data.DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=4,
collate_fn=collate)
train_steps = int(len(train_dataloader) / args.grad_steps * args.epochs)
if args.optimizer == 'AdamW':
no_decay = ["bias", "LayerNorm.weight"]
optimizer_grouped_parameters = [
{
"params": [
p for n, p in model.named_parameters()
if not any(nd in n for nd in no_decay)
],
"weight_decay":
0.0
},
{
"params": [
p for n, p in model.named_parameters()
if any(nd in n for nd in no_decay)
],
"weight_decay":
0.0
},
]
optimizer = AdamW(optimizer_grouped_parameters, lr=args.lr, eps=1e-8)
elif args.optimizer == 'SGD':
optimizer = SGD(model.parameters(), lr=args.lr)
elif args.optimizer == 'Adafactor':
optimizer = Adafactor(model.parameters(), lr=args.lr, beta1=0)
# check
if args.lr_schedule:
scheduler = get_linear_schedule_with_warmup(
optimizer,
num_warmup_steps=int(0.1 * train_steps),
num_training_steps=train_steps)
if os.path.exists(args.checkpoint):
print('Loading optimizer and scheduler')
optimizer.load_state_dict(
torch.load(os.path.join(args.checkpoint, 'optimizer.pt')))
if args.lr_schedule:
scheduler.load_state_dict(
torch.load(os.path.join(args.checkpoint, 'scheduler.pt')))
if args.accelerator == 'GPU' and args.fp16 == True:
from apex import amp
model, optimizer = amp.initialize(model,
optimizer,
opt_level=args.apex_mode,
loss_scale="dynamic")
elif args.accelerator == 'TPU':
import torch_xla.core.xla_model as xm
wandb.watch(model, log='parameters')
gradients = {}
global_step = 0
epochs_trained = 0
steps_trained_in_current_epoch = 0
if os.path.exists(args.checkpoint):
global_step = int(args.checkpoint.split('-')[-1].split('/')[0])
# check
epochs_trained = global_step // (len(train_dataloader) //
args.grad_steps)
steps_trained_in_current_epoch = global_step % (
len(train_dataloader) // args.grad_steps) * args.grad_steps
for epoch in range(epochs_trained, args.epochs):
train_loss = 0
val_loss = 0
print(f"Epoch: {epoch}")
model.train()
for i, batch in tqdm(enumerate(train_dataloader),
total=int(len(train_dataset) / args.batch_size)):
if steps_trained_in_current_epoch > 0:
steps_trained_in_current_epoch -= 1
continue
inputs, labels = batch.to(args.device), batch.to(args.device)
out = model(inputs, labels=labels)
loss = out[0]
loss = loss / args.grad_steps
# check
train_loss += loss.item()
if args.accelerator == 'GPU' and args.fp16 == True:
with amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
else:
loss.backward()
if (i + 1) % args.grad_steps == 0:
# check
if args.accelerator == 'GPU' and args.fp16 == True:
torch.nn.utils.clip_grad_norm_(
amp.master_params(optimizer), 1)
else:
torch.nn.utils.clip_grad_norm_(model.parameters(), 1)
# check
if args.accelerator == 'TPU':
xm.optimizer_step(optimizer, barrier=True)
else:
optimizer.step()
if args.lr_schedule:
scheduler.step()
if global_step % args.logging_steps == 0:
if args.lr_schedule:
lr = optimizer.param_groups[0]['lr']
else:
lr = scheduler.get_last_lr()[0]
wandb.log(
{
"train_loss": loss.item() * args.grad_steps,
"learning_rate": lr
},
step=global_step)
if global_step % args.hist_steps == 0:
for name, param in model.named_parameters():
if param.grad is not None:
try:
gradients[
f"gradients/{name}"] = wandb.Histogram(
param.grad.detach().cpu().numpy())
except:
pass
wandb.log(gradients, step=global_step)
optimizer.zero_grad()
global_step += 1
# Must be in grad_accum block b/c if it is > 0, the model will get saved multiple times
if global_step % args.save_steps == 0:
print(f'Saving model at global step: {global_step}')
checkpoint_dir = os.path.join(args.save_dir,
f'checkpoint-{global_step}')
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
model.save_pretrained(checkpoint_dir)
tokenizer.save_pretrained(checkpoint_dir)
torch.save(optimizer.state_dict(),
os.path.join(checkpoint_dir, 'optimizer.pt'))
if args.lr_schedule:
torch.save(
scheduler.state_dict(),
os.path.join(checkpoint_dir, 'scheduler.pt'))
model.eval()
with torch.no_grad():
for j, batch in tqdm(enumerate(val_dataloader),
total=int(len(val_dataset) /
args.batch_size)):
inputs, labels = batch.to(args.device), batch.to(args.device)
out = model(inputs, labels=labels)
loss = out[0]
val_loss += loss.item()
train_loss /= (i + 1)
val_loss /= (j + 1)
train_loss *= args.grad_steps
train_perplexity = torch.exp(torch.tensor(train_loss))
val_perplexity = torch.exp(torch.tensor(val_loss))
adjusted_train_perplexity = torch.exp(
torch.tensor(train_loss) * ((train_dataset.n_tokens - 1) /
(train_dataset.n_original_tokens - 1)))
adjusted_val_perplexity = torch.exp(
torch.tensor(val_loss) * ((val_dataset.n_tokens - 1) /
(val_dataset.n_original_tokens - 1)))
sample(model, tokenizer, args)
wandb.log(
{
"train_epoch_loss": train_loss,
"train_perplexity": train_perplexity,
"adjusted_train_perplexity": adjusted_train_perplexity,
'val_epoch_loss': val_loss,
'val_perplexity': val_perplexity,
'adjusted_val_perplexity': adjusted_val_perplexity
},
step=global_step)
message = f'Finished epoch {epoch} | Train loss: {train_loss} | Train perplexity: {train_perplexity} | Adjusted Train perplexity: {adjusted_train_perplexity} | Val Loss: {val_loss} | Val Perplexity: {val_perplexity} | Adjusted Val Perplexity: {adjusted_val_perplexity}'
print(message)
model.save_pretrained(args.save_dir)
tokenizer.save_pretrained(args.save_dir)
torch.save(optimizer.state_dict(),
os.path.join(args.save_dir, 'optimizer.pt'))
if args.lr_schedule:
torch.save(scheduler.state_dict(),
os.path.join(args.save_dir, 'scheduler.pt'))
def main():
# See all possible arguments in src/transformers/training_args.py
# or by passing the --help flag to this script.
# We now keep distinct sets of args, for a cleaner separation of concerns.
parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
model_args, data_args, training_args = parser.parse_args_into_dataclasses()
if data_args.eval_data_file is None and training_args.do_eval:
raise ValueError(
"Cannot do evaluation without an evaluation data file. Either supply a file to --eval_data_file "
"or remove the --do_eval argument."
)
if (
os.path.exists(training_args.output_dir)
and os.listdir(training_args.output_dir)
and training_args.do_train
and not training_args.overwrite_output_dir
):
raise ValueError(
f"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
)
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN,
)
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
training_args.local_rank,
training_args.device,
training_args.n_gpu,
bool(training_args.local_rank != -1),
training_args.fp16,
)
logger.info("Training/evaluation parameters %s", training_args)
# Set seed
set_seed(training_args.seed)
# Load pretrained model and tokenizer
#
# Distributed training:
# The .from_pretrained methods guarantee that only one local process can concurrently
# download model & vocab.
if model_args.config_name:
config = AutoConfig.from_pretrained(model_args.config_name, cache_dir=model_args.cache_dir)
elif model_args.model_name_or_path:
config = AutoConfig.from_pretrained(model_args.model_name_or_path, cache_dir=model_args.cache_dir)
else:
config = CONFIG_MAPPING[model_args.model_type]()
logger.warning("You are instantiating a new config instance from scratch.")
if model_args.tokenizer_name:
tokenizer = AutoTokenizer.from_pretrained(model_args.tokenizer_name, cache_dir=model_args.cache_dir)
elif model_args.model_name_or_path:
tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, cache_dir=model_args.cache_dir)
else:
raise ValueError(
"You are instantiating a new tokenizer from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --tokenizer_name"
)
# Dirty hack to add NK vocab to our tokenizer
# From: https://github.com/deepset-ai/FARM/issues/157
from collections import OrderedDict
from transformers import BertTokenizer, WordpieceTokenizer
with open('jobert-vocab.txt', 'r', encoding='utf8') as fp:
vocab = fp.read().splitlines()
tokens_to_add = [token for token in vocab if not (token in tokenizer.vocab or token in tokenizer.all_special_tokens)]
tokenizer.vocab = OrderedDict([
*tokenizer.vocab.items(),
*[
(token, i + len(tokenizer.vocab))
for i, token in enumerate(tokens_to_add)
]
])
tokenizer.ids_to_tokens = OrderedDict([(ids, tok) for tok, ids in tokenizer.vocab.items()])
tokenizer.wordpiece_tokenizer = WordpieceTokenizer(vocab=tokenizer.vocab, unk_token=tokenizer.unk_token)
if model_args.model_name_or_path:
model = AutoModelWithLMHead.from_pretrained(
model_args.model_name_or_path,
from_tf=bool(".ckpt" in model_args.model_name_or_path),
config=config,
cache_dir=model_args.cache_dir,
)
else:
logger.info("Training new model from scratch")
model = AutoModelWithLMHead.from_config(config)
model.resize_token_embeddings(len(tokenizer))
if config.model_type in ["bert", "roberta", "distilbert", "camembert"] and not data_args.mlm:
raise ValueError(
"BERT and RoBERTa-like models do not have LM heads but masked LM heads. They must be run using the"
"--mlm flag (masked language modeling)."
)
if data_args.block_size <= 0:
data_args.block_size = tokenizer.max_len
# Our input block size will be the max possible for the model
else:
data_args.block_size = min(data_args.block_size, tokenizer.max_len)
# Get datasets
train_dataset = (
get_dataset(data_args, tokenizer=tokenizer, cache_dir=model_args.cache_dir) if training_args.do_train else None
)
eval_dataset = (
get_dataset(data_args, tokenizer=tokenizer, evaluate=True, cache_dir=model_args.cache_dir)
if training_args.do_eval
else None
)
if config.model_type == "xlnet":
data_collator = DataCollatorForPermutationLanguageModeling(
tokenizer=tokenizer,
plm_probability=data_args.plm_probability,
max_span_length=data_args.max_span_length,
)
else:
data_collator = DataCollatorForLanguageModeling(
tokenizer=tokenizer, mlm=data_args.mlm, mlm_probability=data_args.mlm_probability
)
# Initialize our Trainer
trainer = Trainer(
model=model,
args=training_args,
data_collator=data_collator,
train_dataset=train_dataset,
eval_dataset=eval_dataset,
prediction_loss_only=True,
)
# Training
if training_args.do_train:
model_path = (
model_args.model_name_or_path
if model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path)
else None
)
trainer.train(model_path=model_path)
trainer.save_model()
# For convenience, we also re-save the tokenizer to the same directory,
# so that you can share your model easily on huggingface.co/models =)
if trainer.is_world_master():
tokenizer.save_pretrained(training_args.output_dir)
# Evaluation
results = {}
if training_args.do_eval:
logger.info("*** Evaluate ***")
eval_output = trainer.evaluate()
perplexity = math.exp(eval_output["eval_loss"])
result = {"perplexity": perplexity}
output_eval_file = os.path.join(training_args.output_dir, "eval_results_lm.txt")
if trainer.is_world_master():
with open(output_eval_file, "w") as writer:
logger.info("***** Eval results *****")
for key in sorted(result.keys()):
logger.info(" %s = %s", key, str(result[key]))
writer.write("%s = %s\n" % (key, str(result[key])))
results.update(result)
return results
def main():
parser = argparse.ArgumentParser()
# Required parameters
parser.add_argument(
"--output_dir",
type=str,
required=True,
help=
"The output directory where the model predictions and checkpoints will be written.",
)
parser.add_argument(
"--model_type",
type=str,
required=True,
help="The model architecture to be trained or fine-tuned.",
)
# Other parameters
parser.add_argument(
"--data_dir",
default=None,
type=str,
)
parser.add_argument(
"--line_by_line",
action="store_true",
help=
"Whether distinct lines of text in the dataset are to be handled as distinct sequences.",
)
parser.add_argument(
"--should_continue",
action="store_true",
help="Whether to continue from latest checkpoint in output_dir")
parser.add_argument(
"--model_name_or_path",
default=None,
type=str,
help=
"The model checkpoint for weights initialization. Leave None if you want to train a model from scratch.",
)
parser.add_argument(
"--language",
default=None,
type=str,
required=True,
help=
"Evaluation language. Also train language if `train_language` is set to None.",
)
parser.add_argument(
"--mlm",
action="store_true",
help=
"Train with masked-language modeling loss instead of language modeling."
)
parser.add_argument(
"--mlm_probability",
type=float,
default=0.15,
help="Ratio of tokens to mask for masked language modeling loss")
parser.add_argument(
"--config_name",
default=None,
type=str,
help=
"Optional pretrained config name or path if not the same as model_name_or_path. If both are None, initialize a new config.",
)
parser.add_argument(
"--tokenizer_name",
default=None,
type=str,
help=
"Optional pretrained tokenizer name or path if not the same as model_name_or_path. If both are None, initialize a new tokenizer.",
)
parser.add_argument(
"--cache_dir",
default=None,
type=str,
help=
"Optional directory to store the pre-trained models downloaded from s3 (instead of the default one)",
)
parser.add_argument(
"--block_size",
default=-1,
type=int,
help="Optional input sequence length after tokenization."
"The training dataset will be truncated in block of this size for training."
"Default to the model max input length for single sentence inputs (take into account special tokens).",
)
parser.add_argument("--do_eval",
action="store_true",
help="Whether to run eval on the dev set.")
parser.add_argument("--per_gpu_eval_batch_size",
default=32,
type=int,
help="Batch size per GPU/CPU for evaluation.")
parser.add_argument("--no_cuda",
action="store_true",
help="Avoid using CUDA when available")
parser.add_argument("--overwrite_output_dir",
action="store_true",
help="Overwrite the content of the output directory")
parser.add_argument(
"--overwrite_cache",
action="store_true",
help="Overwrite the cached training and evaluation sets")
parser.add_argument("--seed",
type=int,
default=42,
help="random seed for initialization")
parser.add_argument(
"--fp16",
action="store_true",
help=
"Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit",
)
parser.add_argument(
"--fp16_opt_level",
type=str,
default="O1",
help=
"For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
"See details at https://nvidia.github.io/apex/amp.html",
)
parser.add_argument("--local_rank",
type=int,
default=-1,
help="For distributed training: local_rank")
parser.add_argument("--server_ip",
type=str,
default="",
help="For distant debugging.")
parser.add_argument("--server_port",
type=str,
default="",
help="For distant debugging.")
parser.add_argument("--target",
type=str,
default="all",
help="target token")
parser.add_argument("--max_sample_size",
type=int,
default=float("inf"),
help="max_sample_size")
args = parser.parse_args()
args.max_sample_size = int(512 / args.block_size) * args.max_sample_size
# Setup CUDA, GPU & distributed training
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available()
and not args.no_cuda else "cpu")
args.n_gpu = 0 if args.no_cuda else torch.cuda.device_count()
else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
torch.distributed.init_process_group(backend="nccl")
args.n_gpu = 1
args.device = device
# Set seed
set_seed(args)
# Load pretrained model and tokenizer
if args.local_rank not in [-1, 0]:
torch.distributed.barrier(
) # Barrier to make sure only the first process in distributed training download model & vocab
if args.config_name:
config = AutoConfig.from_pretrained(args.config_name,
cache_dir=args.cache_dir)
elif args.model_name_or_path:
config = AutoConfig.from_pretrained(
args.model_name_or_path,
cache_dir=args.cache_dir,
output_hidden_states=True,
)
else:
# When we release a pip version exposing CONFIG_MAPPING,
# we can do `config = CONFIG_MAPPING[args.model_type]()`.
raise ValueError(
"You are instantiating a new config instance from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --config_name")
if args.tokenizer_name:
tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_name,
cache_dir=args.cache_dir)
elif args.model_name_or_path:
tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path,
cache_dir=args.cache_dir)
else:
raise ValueError(
"You are instantiating a new tokenizer from scratch. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --tokenizer_name")
if args.block_size <= 0:
args.block_size = tokenizer.max_len
# Our input block size will be the max possible for the model
else:
args.block_size = min(args.block_size, tokenizer.max_len)
if args.model_name_or_path:
model = AutoModelWithLMHead.from_pretrained(
args.model_name_or_path,
from_tf=bool(".ckpt" in args.model_name_or_path),
config=config,
cache_dir=args.cache_dir,
)
else:
logger.info("Training new model from scratch")
model = AutoModelWithLMHead.from_config(config)
model.to(args.device)
if args.local_rank == 0:
torch.distributed.barrier(
) # End of barrier to make sure only the first process in distributed training download model & vocab
logger.info("Training/evaluation parameters %s", args)
# Evaluation
results = {}
if args.do_eval and args.local_rank in [-1, 0]:
logger.info("Extrating Features")
model_name = list(filter(None,
args.model_name_or_path.split("/"))).pop()
extract_feature(args, model, tokenizer,
f"{args.language}_{model_name}")
