class LineByLineTextDataset(Dataset):
def __init__(self, args, file_path: str, block_size=512):
assert os.path.isfile(file_path)
self.block_size = block_size
self.tokenizer = ByteLevelBPETokenizer(
os.path.join(args.tokenizer_name, "vocab.json"),
os.path.join(args.tokenizer_name, "merges.txt"),
)
self.tokenizer._tokenizer.post_processor = RobertaProcessing(
("</s>", self.tokenizer.token_to_id("</s>")),
("<s>", self.tokenizer.token_to_id("<s>")),
)
self.tokenizer.enable_truncation(max_length=block_size)
logger.info("Creating features from dataset file at %s", file_path)
self.examples = []
with open(file_path, encoding="utf-8") as f:
for line in f:
if len(line) > 0 and not line.isspace():
self.examples.append(line)
def __len__(self):
return len(self.examples)
def __getitem__(self, i):
return torch.tensor(self.tokenizer.encode(self.examples[i]).ids[: self.block_size - 2], dtype=torch.long)
def get_tokenizer(path):
tokenizer = ByteLevelBPETokenizer(path + 'vocab.json', path + 'merges.txt')
tokenizer._tokenizer.post_processor = BertProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")),
)
return tokenizer
def test_tokenizer(test_sentence, vocab_path, merge_path):
r"""
Illustrates how the individual Tokenizer works
Args:
test_sentence (:obj:`str`):
Sentence for demonstration purposes
vocab_path (:obj:`str`):
Path where the vocabulary (most frequent tokens ranked by frequency) is saved
merge_path (:obj:`str`):
Path where the merges file is saved
"""
tokenizer = ByteLevelBPETokenizer(vocab_path, merge_path)
tokenizer._tokenizer.post_processor = BertProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")))
tokenizer.enable_truncation(max_length=512)
print("Original sentence " + test_sentence)
print("Encoded string: {}".format(tokenizer.encode(test_sentence).tokens))
encoding = tokenizer.encode(test_sentence)
decoded = tokenizer.decode(encoding.ids)
print("Decoded string: {}".format(decoded))
def __init__(self, evaluate: bool = False):
tokenizer = ByteLevelBPETokenizer(
"./model/bbpe/vocab.json",
"./model/bbpe/merges.txt",
)
tokenizer._tokenizer.post_processor = BertProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")),
)
tokenizer.enable_truncation(max_length=512)
# or use the RobertaTokenizer from `transformers` directly.
self.examples = []
src_files = Path("./data/").glob("*_eval.csv") if evaluate else Path(
"./data/").glob("*_eval.csv")
for src_file in src_files:
print("🔥", src_file)
with open(src_file, 'r', encoding='utf-8') as f:
for index, line in enumerate(f):
self.examples += [
x.ids for x in tokenizer.encode_batch(line)
]
if index % 10000 == 0:
print(src_file, index // 10000)
def __init__(self, cfg):
super().__init__(cfg)
self.scales = [str((cfg.load_size // (2**i))) for i in range(3)]
self.scales.reverse()
self.device_map = {
'style': self.devices[0],
'content': self.devices[0],
'img': self.devices[0]
}
self.network_names = [
'style_model', 'content_model', 'generator', 'discriminators'
]
self.device_name_map = {
'style_model': 'style',
'content_model': 'content',
'generators': 'img',
'discriminators': 'img'
}
tokenizer = ByteLevelBPETokenizer(
"vocab.json",
"merges.txt",
)
tokenizer._tokenizer.post_processor = BertProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")),
)
self.cold = True
self.language_model = LanguageModel(cfg, tokenizer,
self.device_map['style']).to(
self.device_map['style'])
self.content_model = VAE(cfg.rnn_hidden_dim, self.device_map['style'],
cfg).to(self.device_map['style'])
self.style_model = VAE(cfg.rnn_hidden_dim, self.device_map['style'],
cfg).to(self.device_map['style'])
self.generator = StyleGenerator(cfg).to(self.device_map['img'])
self.discriminator = FeatureConvolutionalDiscriminator(cfg).to(
self.device_map['img'])
self.visual_names = ['visual_dict']
self.visual_dict = {'real': None, 'fake': None}
self.loss_names = ['loss']
self.visualizer = Visualizer(cfg)
self.generator_criterion = BinaryCrossEntropyLoss(cfg).to(
self.device_map['img'])
self.consistency_criterion = ColorConsistencyLoss(cfg).to(
self.device_map['img'])
self.distribution_criterion = KLDLoss().to(self.device_map['img'])
self.latent_scale = int(cfg.load_size // (2**6))
self.latent_channels = int(cfg.latent_dim) // (self.latent_scale**2)
self.channels_z = 8 * self.cfg.ngf - self.latent_channels
def train_tokenizer(input_path, output_path, vocab_size=10000):
tokenizer = ByteLevelBPETokenizer()
tokenizer.train(files=[input_path],
vocab_size=vocab_size,
special_tokens=["[PAD]", "<s>", "</s>", "<unk>"])
tokenizer._tokenizer.post_processor = BertProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")),
)
tokenizer.save_model(output_path)
return tokenizer
def __init__(self, max_tokens=512):
## RoBERTa uses BPE tokenizer similar to GPT
t = ByteLevelBPETokenizer("tokenizer/vocab.json",
"tokenizer/merges.txt")
t._tokenizer.post_processor = BertProcessing(
("</s>", t.token_to_id("</s>")),
("<s>", t.token_to_id("<s>")),
)
t.enable_truncation(max_tokens)
t.enable_padding(length=max_tokens, pad_id=t.token_to_id("<pad>"))
self.tokenizer = t
def load_sentence_piece_model():
tokenizer = ByteLevelBPETokenizer(path_vocab, path_model)
tokenizer._tokenizer.post_processor = BertProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")))
tokenizer.enable_truncation(max_length=512)
encoding = tokenizer.encode("배고파요")
print(encoding.tokens)
print(encoding.special_tokens_mask)
print(encoding.ids)
print(encoding.normalized_str)
def load_custom_tokenizer(self, path):
tokenizer = ByteLevelBPETokenizer(path + "-vocab.json",
path + "-merges.txt")
# Add preprocessing tokens like Roberta
tokenizer._tokenizer.post_processor = BertProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")),
)
return PreTrainedTokenizerFast(tokenizer,
pad_token="<pad>",
mask_token="<mask>",
unk_token="<unk>",
bos_token="<s>",
eos_token="</s>")
class HuggingFaceBpeHelper(object):
@staticmethod
def add_cmdline_args(argparser):
parser = argparser.add_argument_group('ByteLevelBPE Arguments')
parser.add_argument('--bpe-vocab',
type=str,
help='path to pre-trained tokenizer vocab')
parser.add_argument('--bpe-merge',
type=str,
help='path to pre-trained tokenizer merge')
parser.add_argument(
'--bpe-add-prefix-space',
type='bool',
hidden=True,
default=True,
help='add prefix space before encoding',
)
return parser
def __init__(self, opt: Opt, shared=None):
try:
from tokenizers import ByteLevelBPETokenizer
except ImportError:
raise ImportError(
'Please install HuggingFace tokenizer with: pip install tokenizers'
)
if 'bpe_vocab' not in opt:
raise ValueError(
'--bpe-vocab is required for loading pretrained tokenizer')
if 'bpe_merge' not in opt:
raise ValueError(
'--bpe-merge is required for loading pretrained tokenizer')
self.vocab_path = opt['bpe_vocab']
self.merge_path = opt['bpe_merge']
if not self.vocab_path or not self.merge_path:
raise IOError('--bpe-vocab and --bpe-merge are mandatory with '
'--dict-tokenizer bytelevelbpe')
if not os.path.isfile(self.vocab_path):
raise IOError(
f'File {self.vocab_path} does not exist. --bpe-vocab must be pretrained.'
)
if not os.path.isfile(self.merge_path):
raise IOError(
f'File {self.merge_path} does not exist. --bpe-merge must be pretrained.'
)
self.add_prefix_space = opt.get('bpe_add_prefix_space', True)
self.tokenizer = ByteLevelBPETokenizer(self.vocab_path,
self.merge_path,
self.add_prefix_space)
def encode(self, text: str) -> List[str]:
return self.tokenizer.encode(text).tokens
def decode(self, x: List[str]) -> str:
return self.tokenizer.decode(self.tokenizer.token_to_id(c) for c in x)
def main(vocab, merges, data_path, lower, save_path):
tokenizer = ByteLevelBPETokenizer(vocab,
merges,
lowercase=lower,
add_prefix_space=True)
sentiment_hash = dict((v[1:], tokenizer.token_to_id(v))
for v in ('Ġpositive', 'Ġnegative', 'Ġneutral'))
print(sentiment_hash)
train = pd.read_csv(os.path.join(data_path, 'train.csv'))
dataset = []
n = nm = 0
score = 0
for line, row in train.iterrows():
if pd.isna(row.text) and pd.isna(row.selected_text): continue
try:
ann = annotate(tokenizer, row.text, row.selected_text.strip(' '))
except AssertionError:
print(row.text, row.selected_text.strip(' '))
continue
ann['sentiment'] = sentiment_hash[row.sentiment]
ann['id'] = row.textID
dataset.append(ann)
decode = ann['text'][
ann['offsets'][ann['start']][0]:ann['offsets'][ann['end']][1]]
if set(decode.split()) != set(ann['gt'].split()):
nm += 1
score += jaccard(decode, ann['gt'])
n += 1
print(f'not match {nm/n}\nBest score {score/n}')
if not lower: save_path = 'cased_' + save_path
joblib.dump(dataset, save_path, compress='zlib')
def __init__(self, evaluate: bool = False):
tokenizer = ByteLevelBPETokenizer(
"models/faberto/vocab.json",
"models/faberto/merges.txt",
)
tokenizer._tokenizer.post_processor = BertProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")),
)
tokenizer.enable_truncation(max_length=512)
# or use the RobertaTokenizer from `transformers` directly.
self.examples = []
src_files = Path("./data/").glob("*-eval.txt") if evaluate else Path(
"./data/").glob("*-train.txt")
for src_file in src_files:
print("🔥", src_file)
lines = src_file.open(encoding="utf-8").read().splitlines()
self.examples += [x.ids for x in tokenizer.encode_batch(lines)]
def inference():
from tokenizers import ByteLevelBPETokenizer
from tokenizers.processors import BertProcessing
'''
initialize tokenizer with saved model files
'''
tokenizer = ByteLevelBPETokenizer(
"./tok_checkpoints/tokenizer_model-vocab.json",
"./tok_checkpoints/tokenizer_model-merges.txt",
)
'''
optional step : preprocess the strings
Ex: add <s> and </s> as BOS and EOS tokens to the string
pad string to some max length and truncate string to some max length
'''
tokenizer._tokenizer.post_processor = BertProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")),
)
tokenizer.enable_padding(pad_token='<pad>',
pad_id=tokenizer.get_vocab()['<pad>'],
length=20)
tokenizer.enable_truncation(max_length=20)
'''
tokenize/encode strings
'''
input_ids = tokenizer.encode("Hello World, Whats up!!!").ids
print("input ids", input_ids)
tokens = tokenizer.encode("Hello World, Whats up!!!").tokens
print("tokens", tokens)
'''
tokenize/encode batch of string
'''
batch_tokenized = tokenizer.encode_batch(
["Hello World, Whats up!!!", "Whata whata wa wada wada"])
input_ids = [i.ids for i in batch_tokenized]
print("input ids", input_ids)
tokens = [i.tokens for i in batch_tokenized]
print("tokens", tokens)
def __init__(self, evaluate: bool = False):
tokenizer = ByteLevelBPETokenizer(
vocab_file,
merges_file
)
tokenizer._tokenizer.post_processor = BertProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")),
)
tokenizer.enable_truncation(max_length=512)
self.examples = []
src_files = Path(data_folder).glob("**/*.txt")
for src_file in src_files:
print("🇩🇰", src_file)
lines = src_file.read_text(encoding="utf-8").splitlines()
self.examples += [x.ids for x in tokenizer.encode_batch(lines)]
class FullTokenizer(object):
"""Runs end-to-end tokenziation."""
def __init__(self, vocab_file, do_lower_case=True):
self.vocab = load_vocab(vocab_file)
self.inv_vocab = {v: k for k, v in self.vocab.items()}
self.tokenizer = ByteLevelBPETokenizer(vocab_file + '/vocab.json',
vocab_file + '/merges.txt')
def tokenize(self, text):
return self.tokenizer.encode(text).ids
def convert_tokens_to_ids(self, tokens):
return [self.tokenizer.token_to_id(tok) for tok in tokens]
def convert_ids_to_tokens(self, ids):
return self.tokenizer.decode(ids)
def load_sentence_piece_model(path_vocab, path_model):
tokenizer = ByteLevelBPETokenizer(path_vocab, path_model)
tokenizer._tokenizer.post_processor = BertProcessing(
("<bos>", tokenizer.token_to_id("<bos>")),
("<eos>", tokenizer.token_to_id("<eos>"))
)
tokenizer.enable_truncation(max_length=512)
# encoding = tokenizer.encode("배고파요")
# print(encoding.tokens)
# print(encoding.special_tokens_mask)
# print(encoding.ids)
# print(encoding.normalized_str)
#
# decoding = tokenizer.decode([2, 1177, 276, 692, 571, 1])
# print(decoding)
return tokenizer
# Customize training
tokenizer.train(files=paths,
vocab_size=52_000,
min_frequency=2,
special_tokens=[
"<s>",
"<pad>",
"</s>",
"<unk>",
"<mask>",
])
# Save files to disk
tokenizer.save(".", "rubinberto")
tokenizer = ByteLevelBPETokenizer(
"rubinberto-vocab.json",
"rubinberto-merges.txt",
)
tokenizer._tokenizer.post_processor = BertProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")),
)
tokenizer.enable_truncation(max_length=512)
print(
tokenizer.encode(
"А можно вспоминать не о событиях, а, например, о чувствах, испытываемых нами за «отчетный период»."
).tokens)
class HuggingFaceBpeHelper(BPEHelper):
"""
HuggingFace's ByteLevelBPE Tokenizer.
Fast because Rust.
"""
def __init__(self, opt: Opt, shared: TShared = None):
super().__init__(opt, shared)
# Default true for HF
self.special_tok_map = {} # map from HF
self.add_prefix_space = opt.get('bpe_add_prefix_space', True)
if self.add_prefix_space is None:
self.add_prefix_space = True
if opt.get('dict_loaded'):
dfname = opt['dict_file']
if PathManager.exists(f'{dfname}-merges.txt'):
opt['bpe_merge'] = f'{dfname}-merges.txt'
if PathManager.exists(f'{dfname}-vocab.json'):
opt['bpe_vocab'] = f'{dfname}-vocab.json'
try:
from tokenizers import ByteLevelBPETokenizer
except ImportError:
raise ImportError(
'Please install HuggingFace tokenizer with: pip install tokenizers'
)
if self.bpe_dropout:
raise NotImplementedError(
'--bpe-dropout is not supported with ByteLevelBPE because tokenizers '
'library does not allow dynamically turning BPE on/off. You can use '
'--dict-tokenizer slow_bytelevel_bpe to gain this feature.'
)
if self.lower:
warn_once('Are you sure you want to lower case your BPE dictionary?')
if self.maxtokens > 0 or self.minfreq > 0:
raise ValueError(
'You should not filter vocabulary with using --dict-tokenizer bytelevelbpe'
' (no --dict-minfreq or --dict-maxtokens).'
)
if 'bpe_vocab' not in opt:
raise ValueError('--bpe-vocab is required for loading pretrained tokenizer')
if 'bpe_merge' not in opt:
raise ValueError('--bpe-merge is required for loading pretrained tokenizer')
self.vocab_path = opt['bpe_vocab']
self.merge_path = opt['bpe_merge']
if not self.vocab_path or not self.merge_path:
raise IOError(
'--bpe-vocab and --bpe-merge are mandatory with '
'--dict-tokenizer bytelevelbpe'
)
if not PathManager.exists(self.vocab_path):
raise IOError(
f'File {self.vocab_path} does not exist. --bpe-vocab must be pretrained.'
)
if not PathManager.exists(self.merge_path):
raise IOError(
f'File {self.merge_path} does not exist. --bpe-merge must be pretrained.'
)
self.tokenizer = ByteLevelBPETokenizer(
self.vocab_path, self.merge_path, self.add_prefix_space
)
def helper_encode(self, text: str) -> List[str]:
"""
Decode list of tokens into text string.
:param tokens:
list of tokens
:param delimiter:
string delimiter for tokens
:return text:
decoded text
"""
return self.tokenizer.encode(text).tokens
def helper_decode(
self, tokens: List[str], token_ids: List[int], delimiter: str
) -> str:
"""
Decode list of tokens into text string.
:param tokens:
list of tokens
:param token_ids:
list of token ids
:param delimiter:
string delimiter for tokens
:return text:
decoded text
"""
text = self.tokenizer.decode(token_ids, skip_special_tokens=False)
return text
def add_special_tokens(self, dict_agent, special_tokens: List[str]):
"""
Add special tokens to the tokenizer and dict_agent.
"""
logging.debug(f'adding the following special tokens: {special_tokens}')
self.tokenizer.add_special_tokens(special_tokens) # add to HF
for tok in special_tokens:
parlai_key = dict_agent[tok]
hf_key = self.tokenizer.token_to_id(tok)
self.special_tok_map[parlai_key] = hf_key
def sync_with_dict(self, dict_agent):
"""
Sync the dictionary agent with Hugging Face tokenizer's BPE dict.
Called only once on initialization.
"""
special_tokens = [
dict_agent.null_token,
dict_agent.start_token,
dict_agent.end_token,
dict_agent.unk_token,
]
self.add_special_tokens(dict_agent, special_tokens)
for i in range(self.tokenizer.get_vocab_size() - len(special_tokens)):
token = self.tokenizer.id_to_token(i)
dict_agent.add_token(token)
# We don't have access to the hugging face word frequency table,
# just set it to 1 instead
dict_agent.freq[token] = 1
def save(self, dir_name: str, file_name: str):
"""
Save appropriate files.
:param dir_name:
directory to save.
:param file_name:
file to save.
"""
self.tokenizer.save_model(dir_name, file_name)
def main():
parser = argparse.ArgumentParser()
# Required parameters
parser.add_argument(
"--train_data_file", default=None, type=str, required=True, help="The input training data file (a text file)."
)
parser.add_argument(
"--output_dir",
type=str,
required=True,
help="The output directory where the model predictions and checkpoints will be written.",
)
parser.add_argument(
"--model_type", type=str, required=True, help="The model architecture to be trained or fine-tuned.",
)
# Other parameters
parser.add_argument(
"--eval_data_file",
default=None,
type=str,
help="An optional input evaluation data file to evaluate the perplexity on (a text file).",
)
parser.add_argument(
"--line_by_line",
action="store_true",
help="Whether distinct lines of text in the dataset are to be handled as distinct sequences.",
)
parser.add_argument(
"--should_continue", action="store_true", help="Whether to continue from latest checkpoint in output_dir"
)
parser.add_argument(
"--model_name_or_path",
default=None,
type=str,
help="The model checkpoint for weights initialization. Leave None if you want to train a model from scratch.",
)
parser.add_argument(
"--mlm", action="store_true", help="Train with masked-language modeling loss instead of language modeling."
)
parser.add_argument(
"--mlm_probability", type=float, default=0.15, help="Ratio of tokens to mask for masked language modeling loss"
)
parser.add_argument(
"--config_name",
default=None,
type=str,
help="Optional pretrained config name or path if not the same as model_name_or_path. If both are None, initialize a new config.",
)
parser.add_argument(
"--tokenizer_name",
default=None,
type=str,
help="Optional pretrained tokenizer name or path if not the same as model_name_or_path. If both are None, initialize a new tokenizer.",
)
parser.add_argument(
"--cache_dir",
default=None,
type=str,
help="Optional directory to store the pre-trained models downloaded from s3 (instead of the default one)",
)
parser.add_argument(
"--block_size",
default=-1,
type=int,
help="Optional input sequence length after tokenization."
"The training dataset will be truncated in block of this size for training."
"Default to the model max input length for single sentence inputs (take into account special tokens).",
)
parser.add_argument("--do_train", action="store_true", help="Whether to run training.")
parser.add_argument("--do_eval", action="store_true", help="Whether to run eval on the dev set.")
parser.add_argument(
"--evaluate_during_training", action="store_true", help="Run evaluation during training at each logging step."
)
parser.add_argument("--per_gpu_train_batch_size", default=4, type=int, help="Batch size per GPU/CPU for training.")
parser.add_argument(
"--per_gpu_eval_batch_size", default=4, type=int, help="Batch size per GPU/CPU for evaluation."
)
parser.add_argument(
"--gradient_accumulation_steps",
type=int,
default=1,
help="Number of updates steps to accumulate before performing a backward/update pass.",
)
parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.")
parser.add_argument("--weight_decay", default=0.0, type=float, help="Weight decay if we apply some.")
parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.")
parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.")
parser.add_argument(
"--num_train_epochs", default=1.0, type=float, help="Total number of training epochs to perform."
)
parser.add_argument(
"--max_steps",
default=-1,
type=int,
help="If > 0: set total number of training steps to perform. Override num_train_epochs.",
)
# parser.add_argument("--warmup_steps", default=0, type=int, help="Linear warmup over warmup_steps.")
parser.add_argument("--pct_warmup", default=0.3, type=float, help="Linear warmup over pct_warmup * total_steps.")
parser.add_argument("--logging_steps", type=int, default=500, help="Log every X updates steps.")
parser.add_argument("--save_steps", type=int, default=500, help="Save checkpoint every X updates steps.")
parser.add_argument(
"--save_total_limit",
type=int,
default=None,
help="Limit the total amount of checkpoints, delete the older checkpoints in the output_dir, does not delete by default",
)
parser.add_argument(
"--eval_all_checkpoints",
action="store_true",
help="Evaluate all checkpoints starting with the same prefix as model_name_or_path ending and ending with step number",
)
parser.add_argument("--no_cuda", action="store_true", help="Avoid using CUDA when available")
parser.add_argument(
"--overwrite_output_dir", action="store_true", help="Overwrite the content of the output directory"
)
parser.add_argument(
"--overwrite_cache", action="store_true", help="Overwrite the cached training and evaluation sets"
)
parser.add_argument("--seed", type=int, default=42, help="random seed for initialization")
parser.add_argument(
"--fp16",
action="store_true",
help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit",
)
parser.add_argument(
"--fp16_opt_level",
type=str,
default="O1",
help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."
"See details at https://nvidia.github.io/apex/amp.html",
)
parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank")
parser.add_argument("--server_ip", type=str, default="", help="For distant debugging.")
parser.add_argument("--server_port", type=str, default="", help="For distant debugging.")
args = parser.parse_args()
if args.model_type in ["bert", "roberta", "distilbert", "camembert"] and not args.mlm:
raise ValueError(
"BERT and RoBERTa-like models do not have LM heads but masked LM heads. They must be run using the --mlm "
"flag (masked language modeling)."
)
if args.eval_data_file is None and args.do_eval:
raise ValueError(
"Cannot do evaluation without an evaluation data file. Either supply a file to --eval_data_file "
"or remove the --do_eval argument."
)
if args.should_continue:
sorted_checkpoints = _sorted_checkpoints(args)
if len(sorted_checkpoints) == 0:
raise ValueError("Used --should_continue but no checkpoint was found in --output_dir.")
else:
args.model_name_or_path = sorted_checkpoints[-1]
if (
os.path.exists(args.output_dir)
and os.listdir(args.output_dir)
and args.do_train
and not args.overwrite_output_dir
):
raise ValueError(
"Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(
args.output_dir
)
)
# Setup distant debugging if needed
if args.server_ip and args.server_port:
# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
import ptvsd
print("Waiting for debugger attach")
ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True)
ptvsd.wait_for_attach()
# Setup CUDA, GPU & distributed training
if args.local_rank == -1 or args.no_cuda:
device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu")
args.n_gpu = 0 if args.no_cuda else torch.cuda.device_count()
else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs
torch.cuda.set_device(args.local_rank)
device = torch.device("cuda", args.local_rank)
torch.distributed.init_process_group(backend="nccl")
args.n_gpu = 1
args.device = device
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN,
)
logger.warning(
"Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
args.local_rank,
device,
args.n_gpu,
bool(args.local_rank != -1),
args.fp16,
)
# Set seed
set_seed(args)
# Load pretrained model and tokenizer
if args.local_rank not in [-1, 0]:
torch.distributed.barrier() # Barrier to make sure only the first process in distributed training download model & vocab
config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
if args.config_name:
config = config_class.from_pretrained(args.config_name, cache_dir=args.cache_dir)
elif args.model_name_or_path:
config = config_class.from_pretrained(args.model_name_or_path, cache_dir=args.cache_dir)
else:
config = config_class()
if args.tokenizer_name:
# tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name, cache_dir=args.cache_dir)
tokenizer = ByteLevelBPETokenizer(
os.path.join(args.tokenizer_name, "vocab.json"),
os.path.join(args.tokenizer_name, "merges.txt")
)
tokenizer._tokenizer.post_processor = BertProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")),
)
tokenizer.enable_truncation(max_length=512)
elif args.model_name_or_path:
tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path, cache_dir=args.cache_dir)
else:
raise ValueError(
"You are instantiating a new {} tokenizer. This is not supported, but you can do it from another script, save it,"
"and load it from here, using --tokenizer_name".format(tokenizer_class.__name__)
)
# if args.block_size <= 0:
# args.block_size = tokenizer.max_len
# # Our input block size will be the max possible for the model
# else:
# args.block_size = min(args.block_size, tokenizer.max_len)
if args.model_name_or_path:
model = model_class.from_pretrained(
args.model_name_or_path,
from_tf=bool(".ckpt" in args.model_name_or_path),
config=config,
cache_dir=args.cache_dir,
)
else:
logger.info("Training new model from scratch")
model = model_class(config=config)
model.to(args.device)
if args.local_rank == 0:
torch.distributed.barrier() # End of barrier to make sure only the first process in distributed training download model & vocab
logger.info("Training/evaluation parameters %s", args)
# Training
if args.do_train:
if args.local_rank not in [-1, 0]:
torch.distributed.barrier() # Barrier to make sure only the first process in distributed training process the dataset, and the others will use the cache
train_dataset = load_and_cache_examples(args, tokenizer, evaluate=False)
if args.local_rank == 0:
torch.distributed.barrier()
global_step, tr_loss = train(args, train_dataset, model, tokenizer)
logger.info(" global_step = %s, average loss = %s", global_step, tr_loss)
# Saving best-practices: if you use save_pretrained for the model and tokenizer, you can reload them using from_pretrained()
if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0):
# Create output directory if needed
if args.local_rank in [-1, 0]:
os.makedirs(args.output_dir, exist_ok=True)
logger.info("Saving model checkpoint to %s", args.output_dir)
# Save a trained model, configuration and tokenizer using `save_pretrained()`.
# They can then be reloaded using `from_pretrained()`
model_to_save = (
model.module if hasattr(model, "module") else model
) # Take care of distributed/parallel training
model_to_save.save_pretrained(args.output_dir)
tokenizer.save_pretrained(args.output_dir)
# Good practice: save your training arguments together with the trained model
torch.save(args, os.path.join(args.output_dir, "training_args.bin"))
# Load a trained model and vocabulary that you have fine-tuned
model = model_class.from_pretrained(args.output_dir)
tokenizer = tokenizer_class.from_pretrained(args.output_dir)
model.to(args.device)
# Evaluation
results = {}
if args.do_eval and args.local_rank in [-1, 0]:
checkpoints = [args.output_dir]
if args.eval_all_checkpoints:
checkpoints = list(
os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + "/**/" + WEIGHTS_NAME, recursive=True))
)
logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN) # Reduce logging
logger.info("Evaluate the following checkpoints: %s", checkpoints)
for checkpoint in checkpoints:
global_step = checkpoint.split("-")[-1] if len(checkpoints) > 1 else ""
prefix = checkpoint.split("/")[-1] if checkpoint.find("checkpoint") != -1 else ""
model = model_class.from_pretrained(checkpoint)
model.to(args.device)
result = evaluate(args, model, tokenizer, prefix=prefix)
result = dict((k + "_{}".format(global_step), v) for k, v in result.items())
results.update(result)
return results
def __init__(self, cfg):
super().__init__(cfg)
self.scales = [str((cfg.load_size // (2**i))) for i in range(3)]
self.scales.reverse()
self.device_map = {
'style': self.devices[0],
'content': self.devices[0],
'img': self.devices[0]
}
self.network_names = [
'style_model', 'content_model', 'generator', 'discriminators'
]
self.device_name_map = {
'style_model': 'style',
'content_model': 'content',
'generators': 'img',
'discriminators': 'img'
}
tokenizer = ByteLevelBPETokenizer(
"vocab.json",
"merges.txt",
)
tokenizer._tokenizer.post_processor = BertProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")),
)
self.style_l_model = LanguageModel(cfg, tokenizer,
self.device_map['style']).to(
self.device_map['style'])
self.style_f_model = VAE(cfg.rnn_hidden_dim * 2,
self.device_map['style'],
cfg).to(self.device_map['style'])
self.content_l_model = LanguageModel(cfg, tokenizer,
self.device_map['content']).to(
self.device_map['content'])
self.content_f_model = VAE(cfg.rnn_hidden_dim * 2,
self.device_map['content'],
cfg).to(self.device_map['content'])
self.generator = StyleGenerator(cfg).to(self.device_map['img'])
self.discriminators = {}
for scale in self.scales:
self.discriminators[scale] = PatchConvolutionalDiscriminator(
int(scale), self.device_map['img'], cfg)
self.discriminators = nn.ModuleDict(self.discriminators).to(
self.device_map['img'])
self.visual_names = ['visual_dict']
self.visual_dict = {'real': None, 'fake': None}
self.loss_names = ['loss']
self.visualizer = Visualizer(cfg)
self.image_pool = {scale: ImagePool(cfg) for scale in self.scales}
self.consistency_criterion = ColorConsistencyLoss(cfg).to(
self.device_map['img'])
self.distribution_criterion = KLDLoss().to(self.device_map['img'])
self.generator_criterion = BinaryCrossEntropyLoss(cfg).to(
self.device_map['img'])
class BERTokenizer:
'''
The basic element of preprocessing would be a pretrained tokenizer.
This is a wrapper on top of such tokenizers to support preprocessing with spacy/stanza.
Currently support: BERT, Electra
TODO: Robera has not been support yet because its way of hanlding space
'''
sp_nlp = None
def __init__(self, version):
# download vocab files
cache = os.path.join(os.environ.get("CACHE_DIR", os.getcwd()),
".vector_cache")
vocab_dir = os.path.join(cache, f"{version}")
if not os.path.exists(vocab_dir):
pretrained_tokenizer = AutoTokenizer.from_pretrained(version)
pretrained_tokenizer.save_pretrained(vocab_dir)
if "uncased" in version or "cased" not in version:
lowercase = True # roberta, electra, bert-base-uncased
else:
lowercase = False # bert-cased
if version.startswith("bert") or "electra" in version:
vocab_path = os.path.join(vocab_dir, "vocab.txt")
self.tokenizer = BertWordPieceTokenizer(vocab_path,
lowercase=lowercase)
elif version.startswith("roberta"):
vocab_path = os.path.join(vocab_dir, "vocab.json")
merge_path = os.path.join(vocab_dir, "merges.txt")
self.tokenizer = ByteLevelBPETokenizer(vocab_path,
merge_path,
lowercase=lowercase)
else:
raise NotImplementedError
self.cls_token = self.tokenizer._parameters["cls_token"]
self.cls_token_id = self.tokenizer.token_to_id(self.cls_token)
self.sep_token = self.tokenizer._parameters["sep_token"]
self.sep_token_id = self.tokenizer.token_to_id(self.sep_token)
self.pad_token = self.tokenizer._parameters["pad_token"]
self.pad_token_id = self.tokenizer.token_to_id(self.pad_token)
def _encode(self, input_):
if isinstance(input_, list) or isinstance(input_, tuple):
encodes = self.tokenizer.encode(input_, is_pretokenized=True)
else:
encodes = self.tokenizer.encode(input_)
return encodes
def tokenize(self, text):
encodes = self._encode(text)
tokens = encodes.tokens[1:-1]
return tokens
def tokenize_and_lemmatize(self, text, lang="en"):
"""
This will be used for matching
1) remove cls and sep
2) lemmatize
"""
if BERTokenizer.sp_nlp is None:
snlp = stanza.Pipeline(lang=lang,
use_gpu=True,
tokenize_pretokenized=True)
BERTokenizer.sp_nlp = StanzaLanguage(snlp)
encodes = self._encode(text)
tokens = encodes.tokens[1:-1]
norm_tokens = [t.lemma_ for t in self.sp_nlp([tokens])]
return norm_tokens
def tokenize_with_orig(self, text):
"""
Tokenize but return the original chars, this would be helpful for copying operations.
"""
# TODO: if text is a list, change accordingly how the offset is computed
assert isinstance(text, str)
encodes = self._encode(text)
orig_tokens = [text[i:j] for i, j in encodes.offsets[1:-1]]
return orig_tokens
def tokenize_and_spacy(self, text, lang="en"):
"""
Keep meta information from spacy, used for matching
"""
if BERTokenizer.sp_nlp is None:
snlp = stanza.Pipeline(lang=lang,
use_gpu=True,
tokenize_pretokenized=True)
BERTokenizer.sp_nlp = StanzaLanguage(snlp)
tokens = self.tokenizer.encode(text).tokens[1:-1]
return self.sp_nlp([tokens])
def check_bert_input_seq(self, toks):
if toks[0] == self.cls_token_id and toks[-1] == self.sep_token_id:
return True
else:
return False
def pieces_to_words(self, pieces):
"""
TODO: use general variable of prefix
"""
words = []
cur_word = None
for piece in pieces:
if piece.startswith("##"):
cur_word = cur_word + piece[2:]
else:
if cur_word is not None:
words.append(cur_word)
cur_word = piece
return words
def text_to_ids(self, sent, cls=True):
"""
This function is primarily used convert text to bpe token ids
"""
encs = self._encode(sent)
if cls:
return encs.ids
else:
assert encs.tokens[0] == self.cls_token
return encs.ids[1:] # remove CLS
def pad_sequence_for_bert_batch(self, tokens_lists):
"""
1) Pad with pad token
2) Generate token_type_list
"""
pad_id = self.pad_token_id
max_len = max([len(it) for it in tokens_lists])
assert max_len <= 512
toks_ids = []
att_masks = []
tok_type_lists = []
for item_toks in tokens_lists:
padded_item_toks = item_toks + [pad_id
] * (max_len - len(item_toks))
toks_ids.append(padded_item_toks)
_att_mask = [1] * len(item_toks) + [0] * (max_len - len(item_toks))
att_masks.append(_att_mask)
first_sep_id = padded_item_toks.index(self.sep_token_id)
assert first_sep_id > 0
_tok_type_list = [0] * (first_sep_id +
1) + [1] * (max_len - first_sep_id - 1)
tok_type_lists.append(_tok_type_list)
return toks_ids, att_masks, tok_type_lists
class Tokenizer:
def __init__(self,
model_name,
vocab_file,
*,
merges_file=None,
lowercase=True,
handle_chinese_chars=False,
dropout=None):
self.model_name = model_name
if model_name == 'bert':
self._pad_token = '[PAD]'
self._sep_token = '[SEP]'
self._cls_token = '[CLS]'
self._unk_token = '[UNK]'
if dropout is not None:
logger.warning(
'BPE dropout is not supported by BertWordPieceTokenizer.')
self.tokenizer = BertWordPieceTokenizer(
vocab_file,
lowercase=lowercase,
handle_chinese_chars=handle_chinese_chars,
unk_token=self.unk_token,
cls_token=self.cls_token,
sep_token=self.sep_token)
elif model_name == 'roberta':
if merges_file is None:
raise AttributeError(
'To use ByteLevelTokenizer, specify path to merges file.')
self._pad_token = '<pad>'
self._sep_token = '</s>'
self._cls_token = '<s>'
self._unk_token = '<unk>'
try:
self.tokenizer = ByteLevelBPETokenizer(vocab_file=vocab_file,
merges_file=merges_file,
dropout=dropout)
except TypeError as e:
logger.warning(
'BPE dropout is not supported by ByteLevelBPETokenizer.')
logger.error(e)
self.tokenizer = ByteLevelBPETokenizer(vocab_file=vocab_file,
merges_file=merges_file)
else:
raise NotImplementedError(
f'Tokenizer initialization for model {model_name} is not implemented.'
)
def __len__(self):
return self.tokenizer._tokenizer.get_vocab_size()
def encode(self, string):
return self.tokenizer.encode(string).ids
def decode(self, ids, *, skip_special_tokens=True):
return self.tokenizer.decode(
ids, skip_special_tokens=skip_special_tokens).replace(' ##', '')
@property
def pad_token_id(self):
return self.tokenizer.token_to_id(self._pad_token)
@property
def sep_token_id(self):
return self.tokenizer.token_to_id(self._sep_token)
@property
def cls_token_id(self):
return self.tokenizer.token_to_id(self._cls_token)
@property
def unk_token_id(self):
return self.tokenizer.token_to_id(self._unk_token)
@property
def pad_token(self):
return self._pad_token
@property
def sep_token(self):
return self._sep_token
@property
def cls_token(self):
return self._cls_token
@property
def unk_token(self):
return self._unk_token
class HuggingfaceTokenizerBPE(nn.Module):
def __init__(self, text_files, dataset_info_path='', config_data=None):
super().__init__()
# The default vocab size in the BERT model is 30522. If we want a number larger than that, we will also have to
# change the BERT configuration.
vocab_size = 30000
self.info = f'hug{vocab_size}'
with open(f'config/data/{config_data}.json') as json_file:
tokenizer_from = json.load(json_file)['tokenizer_from']
config_name = config_data if tokenizer_from == "" else tokenizer_from
print(
os.path.join(dataset_info_path,
f'tokenizer_{config_name}_{vocab_size}-vocab.json'))
# The loading is only properly implemented starting from version 0.8. However, it makes the system use a lot of
# CPU for no reason (it is much slower). Maybe it will be fixed in the future.
if not os.path.isfile(
os.path.join(
dataset_info_path,
f'tokenizer_{config_name}_{vocab_size}-vocab.json')):
text_files = text_files()
self.tokenizer = ByteLevelBPETokenizer()
# Join into a single file. This should NOT be necessary but it does not work properly with a lot of files
with open('/tmp/text_files.txt', 'wb') as outfile:
for filename in tqdm(
text_files,
desc='Joining all files into one for tokenization'):
with open(filename, 'rb') as readfile:
shutil.copyfileobj(readfile, outfile)
text_files = '/tmp/text_files.txt'
self.tokenizer.train(text_files,
vocab_size=vocab_size,
special_tokens=special_tokens)
self.tokenizer.save(dataset_info_path,
f'tokenizer_{config_name}_{vocab_size}')
# No "else", always load for consistency
vocab_file = os.path.join(
dataset_info_path,
f'tokenizer_{config_name}_{vocab_size}-vocab.json')
merges_file = os.path.join(
dataset_info_path,
f'tokenizer_{config_name}_{vocab_size}-merges.txt')
self.tokenizer = ByteLevelBPETokenizer(vocab_file=vocab_file,
merges_file=merges_file)
self.tokenizer.add_special_tokens(special_tokens)
self.index_special_tokens = {
tok: self.tokenizer.encode(tok).ids[0]
for tok in special_tokens
}
@property
def device(self):
return self._float_tensor.device
def encode(self, sentence: str):
output = self.tokenizer.encode(sentence)
token_ids = output.ids
tokens = output.tokens
return torch.tensor(token_ids), tokens
def decode(self, tokens: torch.LongTensor):
assert tokens.dim() == 1
tokens = list(tokens.cpu().numpy())
sentences = self.tokenizer.decode(tokens)
return sentences
def id_to_token(self, token_id):
if type(token_id) != torch.Tensor:
token_id = torch.tensor(token_id)
return self.tokenizer.id_to_token(token_id)
def token_to_id(self, token):
assert type(token) == str
return self.tokenizer.token_to_id(token)
def __len__(self):
return self.tokenizer.get_vocab_size()
# This is simply for PyCharm to find the correct reference to the methods of the class
def __call__(self, *input, **kwargs) -> typing.Any:
return super().__call__(*input, **kwargs)
