class LineByLineTextDataset(Dataset): def __init__(self, args, file_path: str, block_size=512): assert os.path.isfile(file_path) self.block_size = block_size self.tokenizer = ByteLevelBPETokenizer( os.path.join(args.tokenizer_name, "vocab.json"), os.path.join(args.tokenizer_name, "merges.txt"), ) self.tokenizer._tokenizer.post_processor = RobertaProcessing( ("</s>", self.tokenizer.token_to_id("</s>")), ("<s>", self.tokenizer.token_to_id("<s>")), ) self.tokenizer.enable_truncation(max_length=block_size) logger.info("Creating features from dataset file at %s", file_path) self.examples = [] with open(file_path, encoding="utf-8") as f: for line in f: if len(line) > 0 and not line.isspace(): self.examples.append(line) def __len__(self): return len(self.examples) def __getitem__(self, i): return torch.tensor(self.tokenizer.encode(self.examples[i]).ids[: self.block_size - 2], dtype=torch.long)
def get_tokenizer(path): tokenizer = ByteLevelBPETokenizer(path + 'vocab.json', path + 'merges.txt') tokenizer._tokenizer.post_processor = BertProcessing( ("</s>", tokenizer.token_to_id("</s>")), ("<s>", tokenizer.token_to_id("<s>")), ) return tokenizer
def test_tokenizer(test_sentence, vocab_path, merge_path): r""" Illustrates how the individual Tokenizer works Args: test_sentence (:obj:`str`): Sentence for demonstration purposes vocab_path (:obj:`str`): Path where the vocabulary (most frequent tokens ranked by frequency) is saved merge_path (:obj:`str`): Path where the merges file is saved """ tokenizer = ByteLevelBPETokenizer(vocab_path, merge_path) tokenizer._tokenizer.post_processor = BertProcessing( ("</s>", tokenizer.token_to_id("</s>")), ("<s>", tokenizer.token_to_id("<s>"))) tokenizer.enable_truncation(max_length=512) print("Original sentence " + test_sentence) print("Encoded string: {}".format(tokenizer.encode(test_sentence).tokens)) encoding = tokenizer.encode(test_sentence) decoded = tokenizer.decode(encoding.ids) print("Decoded string: {}".format(decoded))
def __init__(self, evaluate: bool = False): tokenizer = ByteLevelBPETokenizer( "./model/bbpe/vocab.json", "./model/bbpe/merges.txt", ) tokenizer._tokenizer.post_processor = BertProcessing( ("</s>", tokenizer.token_to_id("</s>")), ("<s>", tokenizer.token_to_id("<s>")), ) tokenizer.enable_truncation(max_length=512) # or use the RobertaTokenizer from `transformers` directly. self.examples = [] src_files = Path("./data/").glob("*_eval.csv") if evaluate else Path( "./data/").glob("*_eval.csv") for src_file in src_files: print("🔥", src_file) with open(src_file, 'r', encoding='utf-8') as f: for index, line in enumerate(f): self.examples += [ x.ids for x in tokenizer.encode_batch(line) ] if index % 10000 == 0: print(src_file, index // 10000)
def __init__(self, cfg): super().__init__(cfg) self.scales = [str((cfg.load_size // (2**i))) for i in range(3)] self.scales.reverse() self.device_map = { 'style': self.devices[0], 'content': self.devices[0], 'img': self.devices[0] } self.network_names = [ 'style_model', 'content_model', 'generator', 'discriminators' ] self.device_name_map = { 'style_model': 'style', 'content_model': 'content', 'generators': 'img', 'discriminators': 'img' } tokenizer = ByteLevelBPETokenizer( "vocab.json", "merges.txt", ) tokenizer._tokenizer.post_processor = BertProcessing( ("</s>", tokenizer.token_to_id("</s>")), ("<s>", tokenizer.token_to_id("<s>")), ) self.cold = True self.language_model = LanguageModel(cfg, tokenizer, self.device_map['style']).to( self.device_map['style']) self.content_model = VAE(cfg.rnn_hidden_dim, self.device_map['style'], cfg).to(self.device_map['style']) self.style_model = VAE(cfg.rnn_hidden_dim, self.device_map['style'], cfg).to(self.device_map['style']) self.generator = StyleGenerator(cfg).to(self.device_map['img']) self.discriminator = FeatureConvolutionalDiscriminator(cfg).to( self.device_map['img']) self.visual_names = ['visual_dict'] self.visual_dict = {'real': None, 'fake': None} self.loss_names = ['loss'] self.visualizer = Visualizer(cfg) self.generator_criterion = BinaryCrossEntropyLoss(cfg).to( self.device_map['img']) self.consistency_criterion = ColorConsistencyLoss(cfg).to( self.device_map['img']) self.distribution_criterion = KLDLoss().to(self.device_map['img']) self.latent_scale = int(cfg.load_size // (2**6)) self.latent_channels = int(cfg.latent_dim) // (self.latent_scale**2) self.channels_z = 8 * self.cfg.ngf - self.latent_channels
def train_tokenizer(input_path, output_path, vocab_size=10000): tokenizer = ByteLevelBPETokenizer() tokenizer.train(files=[input_path], vocab_size=vocab_size, special_tokens=["[PAD]", "<s>", "</s>", "<unk>"]) tokenizer._tokenizer.post_processor = BertProcessing( ("</s>", tokenizer.token_to_id("</s>")), ("<s>", tokenizer.token_to_id("<s>")), ) tokenizer.save_model(output_path) return tokenizer
def __init__(self, max_tokens=512): ## RoBERTa uses BPE tokenizer similar to GPT t = ByteLevelBPETokenizer("tokenizer/vocab.json", "tokenizer/merges.txt") t._tokenizer.post_processor = BertProcessing( ("</s>", t.token_to_id("</s>")), ("<s>", t.token_to_id("<s>")), ) t.enable_truncation(max_tokens) t.enable_padding(length=max_tokens, pad_id=t.token_to_id("<pad>")) self.tokenizer = t
def load_sentence_piece_model(): tokenizer = ByteLevelBPETokenizer(path_vocab, path_model) tokenizer._tokenizer.post_processor = BertProcessing( ("</s>", tokenizer.token_to_id("</s>")), ("<s>", tokenizer.token_to_id("<s>"))) tokenizer.enable_truncation(max_length=512) encoding = tokenizer.encode("배고파요") print(encoding.tokens) print(encoding.special_tokens_mask) print(encoding.ids) print(encoding.normalized_str)
def load_custom_tokenizer(self, path): tokenizer = ByteLevelBPETokenizer(path + "-vocab.json", path + "-merges.txt") # Add preprocessing tokens like Roberta tokenizer._tokenizer.post_processor = BertProcessing( ("</s>", tokenizer.token_to_id("</s>")), ("<s>", tokenizer.token_to_id("<s>")), ) return PreTrainedTokenizerFast(tokenizer, pad_token="<pad>", mask_token="<mask>", unk_token="<unk>", bos_token="<s>", eos_token="</s>")
class HuggingFaceBpeHelper(object): @staticmethod def add_cmdline_args(argparser): parser = argparser.add_argument_group('ByteLevelBPE Arguments') parser.add_argument('--bpe-vocab', type=str, help='path to pre-trained tokenizer vocab') parser.add_argument('--bpe-merge', type=str, help='path to pre-trained tokenizer merge') parser.add_argument( '--bpe-add-prefix-space', type='bool', hidden=True, default=True, help='add prefix space before encoding', ) return parser def __init__(self, opt: Opt, shared=None): try: from tokenizers import ByteLevelBPETokenizer except ImportError: raise ImportError( 'Please install HuggingFace tokenizer with: pip install tokenizers' ) if 'bpe_vocab' not in opt: raise ValueError( '--bpe-vocab is required for loading pretrained tokenizer') if 'bpe_merge' not in opt: raise ValueError( '--bpe-merge is required for loading pretrained tokenizer') self.vocab_path = opt['bpe_vocab'] self.merge_path = opt['bpe_merge'] if not self.vocab_path or not self.merge_path: raise IOError('--bpe-vocab and --bpe-merge are mandatory with ' '--dict-tokenizer bytelevelbpe') if not os.path.isfile(self.vocab_path): raise IOError( f'File {self.vocab_path} does not exist. --bpe-vocab must be pretrained.' ) if not os.path.isfile(self.merge_path): raise IOError( f'File {self.merge_path} does not exist. --bpe-merge must be pretrained.' ) self.add_prefix_space = opt.get('bpe_add_prefix_space', True) self.tokenizer = ByteLevelBPETokenizer(self.vocab_path, self.merge_path, self.add_prefix_space) def encode(self, text: str) -> List[str]: return self.tokenizer.encode(text).tokens def decode(self, x: List[str]) -> str: return self.tokenizer.decode(self.tokenizer.token_to_id(c) for c in x)
def main(vocab, merges, data_path, lower, save_path): tokenizer = ByteLevelBPETokenizer(vocab, merges, lowercase=lower, add_prefix_space=True) sentiment_hash = dict((v[1:], tokenizer.token_to_id(v)) for v in ('Ġpositive', 'Ġnegative', 'Ġneutral')) print(sentiment_hash) train = pd.read_csv(os.path.join(data_path, 'train.csv')) dataset = [] n = nm = 0 score = 0 for line, row in train.iterrows(): if pd.isna(row.text) and pd.isna(row.selected_text): continue try: ann = annotate(tokenizer, row.text, row.selected_text.strip(' ')) except AssertionError: print(row.text, row.selected_text.strip(' ')) continue ann['sentiment'] = sentiment_hash[row.sentiment] ann['id'] = row.textID dataset.append(ann) decode = ann['text'][ ann['offsets'][ann['start']][0]:ann['offsets'][ann['end']][1]] if set(decode.split()) != set(ann['gt'].split()): nm += 1 score += jaccard(decode, ann['gt']) n += 1 print(f'not match {nm/n}\nBest score {score/n}') if not lower: save_path = 'cased_' + save_path joblib.dump(dataset, save_path, compress='zlib')
def __init__(self, evaluate: bool = False): tokenizer = ByteLevelBPETokenizer( "models/faberto/vocab.json", "models/faberto/merges.txt", ) tokenizer._tokenizer.post_processor = BertProcessing( ("</s>", tokenizer.token_to_id("</s>")), ("<s>", tokenizer.token_to_id("<s>")), ) tokenizer.enable_truncation(max_length=512) # or use the RobertaTokenizer from `transformers` directly. self.examples = [] src_files = Path("./data/").glob("*-eval.txt") if evaluate else Path( "./data/").glob("*-train.txt") for src_file in src_files: print("🔥", src_file) lines = src_file.open(encoding="utf-8").read().splitlines() self.examples += [x.ids for x in tokenizer.encode_batch(lines)]
def inference(): from tokenizers import ByteLevelBPETokenizer from tokenizers.processors import BertProcessing ''' initialize tokenizer with saved model files ''' tokenizer = ByteLevelBPETokenizer( "./tok_checkpoints/tokenizer_model-vocab.json", "./tok_checkpoints/tokenizer_model-merges.txt", ) ''' optional step : preprocess the strings Ex: add <s> and </s> as BOS and EOS tokens to the string pad string to some max length and truncate string to some max length ''' tokenizer._tokenizer.post_processor = BertProcessing( ("</s>", tokenizer.token_to_id("</s>")), ("<s>", tokenizer.token_to_id("<s>")), ) tokenizer.enable_padding(pad_token='<pad>', pad_id=tokenizer.get_vocab()['<pad>'], length=20) tokenizer.enable_truncation(max_length=20) ''' tokenize/encode strings ''' input_ids = tokenizer.encode("Hello World, Whats up!!!").ids print("input ids", input_ids) tokens = tokenizer.encode("Hello World, Whats up!!!").tokens print("tokens", tokens) ''' tokenize/encode batch of string ''' batch_tokenized = tokenizer.encode_batch( ["Hello World, Whats up!!!", "Whata whata wa wada wada"]) input_ids = [i.ids for i in batch_tokenized] print("input ids", input_ids) tokens = [i.tokens for i in batch_tokenized] print("tokens", tokens)
def __init__(self, evaluate: bool = False): tokenizer = ByteLevelBPETokenizer( vocab_file, merges_file ) tokenizer._tokenizer.post_processor = BertProcessing( ("</s>", tokenizer.token_to_id("</s>")), ("<s>", tokenizer.token_to_id("<s>")), ) tokenizer.enable_truncation(max_length=512) self.examples = [] src_files = Path(data_folder).glob("**/*.txt") for src_file in src_files: print("🇩🇰", src_file) lines = src_file.read_text(encoding="utf-8").splitlines() self.examples += [x.ids for x in tokenizer.encode_batch(lines)]
class FullTokenizer(object): """Runs end-to-end tokenziation.""" def __init__(self, vocab_file, do_lower_case=True): self.vocab = load_vocab(vocab_file) self.inv_vocab = {v: k for k, v in self.vocab.items()} self.tokenizer = ByteLevelBPETokenizer(vocab_file + '/vocab.json', vocab_file + '/merges.txt') def tokenize(self, text): return self.tokenizer.encode(text).ids def convert_tokens_to_ids(self, tokens): return [self.tokenizer.token_to_id(tok) for tok in tokens] def convert_ids_to_tokens(self, ids): return self.tokenizer.decode(ids)
def load_sentence_piece_model(path_vocab, path_model): tokenizer = ByteLevelBPETokenizer(path_vocab, path_model) tokenizer._tokenizer.post_processor = BertProcessing( ("<bos>", tokenizer.token_to_id("<bos>")), ("<eos>", tokenizer.token_to_id("<eos>")) ) tokenizer.enable_truncation(max_length=512) # encoding = tokenizer.encode("배고파요") # print(encoding.tokens) # print(encoding.special_tokens_mask) # print(encoding.ids) # print(encoding.normalized_str) # # decoding = tokenizer.decode([2, 1177, 276, 692, 571, 1]) # print(decoding) return tokenizer
# Customize training tokenizer.train(files=paths, vocab_size=52_000, min_frequency=2, special_tokens=[ "<s>", "<pad>", "</s>", "<unk>", "<mask>", ]) # Save files to disk tokenizer.save(".", "rubinberto") tokenizer = ByteLevelBPETokenizer( "rubinberto-vocab.json", "rubinberto-merges.txt", ) tokenizer._tokenizer.post_processor = BertProcessing( ("</s>", tokenizer.token_to_id("</s>")), ("<s>", tokenizer.token_to_id("<s>")), ) tokenizer.enable_truncation(max_length=512) print( tokenizer.encode( "А можно вспоминать не о событиях, а, например, о чувствах, испытываемых нами за «отчетный период»." ).tokens)
class HuggingFaceBpeHelper(BPEHelper): """ HuggingFace's ByteLevelBPE Tokenizer. Fast because Rust. """ def __init__(self, opt: Opt, shared: TShared = None): super().__init__(opt, shared) # Default true for HF self.special_tok_map = {} # map from HF self.add_prefix_space = opt.get('bpe_add_prefix_space', True) if self.add_prefix_space is None: self.add_prefix_space = True if opt.get('dict_loaded'): dfname = opt['dict_file'] if PathManager.exists(f'{dfname}-merges.txt'): opt['bpe_merge'] = f'{dfname}-merges.txt' if PathManager.exists(f'{dfname}-vocab.json'): opt['bpe_vocab'] = f'{dfname}-vocab.json' try: from tokenizers import ByteLevelBPETokenizer except ImportError: raise ImportError( 'Please install HuggingFace tokenizer with: pip install tokenizers' ) if self.bpe_dropout: raise NotImplementedError( '--bpe-dropout is not supported with ByteLevelBPE because tokenizers ' 'library does not allow dynamically turning BPE on/off. You can use ' '--dict-tokenizer slow_bytelevel_bpe to gain this feature.' ) if self.lower: warn_once('Are you sure you want to lower case your BPE dictionary?') if self.maxtokens > 0 or self.minfreq > 0: raise ValueError( 'You should not filter vocabulary with using --dict-tokenizer bytelevelbpe' ' (no --dict-minfreq or --dict-maxtokens).' ) if 'bpe_vocab' not in opt: raise ValueError('--bpe-vocab is required for loading pretrained tokenizer') if 'bpe_merge' not in opt: raise ValueError('--bpe-merge is required for loading pretrained tokenizer') self.vocab_path = opt['bpe_vocab'] self.merge_path = opt['bpe_merge'] if not self.vocab_path or not self.merge_path: raise IOError( '--bpe-vocab and --bpe-merge are mandatory with ' '--dict-tokenizer bytelevelbpe' ) if not PathManager.exists(self.vocab_path): raise IOError( f'File {self.vocab_path} does not exist. --bpe-vocab must be pretrained.' ) if not PathManager.exists(self.merge_path): raise IOError( f'File {self.merge_path} does not exist. --bpe-merge must be pretrained.' ) self.tokenizer = ByteLevelBPETokenizer( self.vocab_path, self.merge_path, self.add_prefix_space ) def helper_encode(self, text: str) -> List[str]: """ Decode list of tokens into text string. :param tokens: list of tokens :param delimiter: string delimiter for tokens :return text: decoded text """ return self.tokenizer.encode(text).tokens def helper_decode( self, tokens: List[str], token_ids: List[int], delimiter: str ) -> str: """ Decode list of tokens into text string. :param tokens: list of tokens :param token_ids: list of token ids :param delimiter: string delimiter for tokens :return text: decoded text """ text = self.tokenizer.decode(token_ids, skip_special_tokens=False) return text def add_special_tokens(self, dict_agent, special_tokens: List[str]): """ Add special tokens to the tokenizer and dict_agent. """ logging.debug(f'adding the following special tokens: {special_tokens}') self.tokenizer.add_special_tokens(special_tokens) # add to HF for tok in special_tokens: parlai_key = dict_agent[tok] hf_key = self.tokenizer.token_to_id(tok) self.special_tok_map[parlai_key] = hf_key def sync_with_dict(self, dict_agent): """ Sync the dictionary agent with Hugging Face tokenizer's BPE dict. Called only once on initialization. """ special_tokens = [ dict_agent.null_token, dict_agent.start_token, dict_agent.end_token, dict_agent.unk_token, ] self.add_special_tokens(dict_agent, special_tokens) for i in range(self.tokenizer.get_vocab_size() - len(special_tokens)): token = self.tokenizer.id_to_token(i) dict_agent.add_token(token) # We don't have access to the hugging face word frequency table, # just set it to 1 instead dict_agent.freq[token] = 1 def save(self, dir_name: str, file_name: str): """ Save appropriate files. :param dir_name: directory to save. :param file_name: file to save. """ self.tokenizer.save_model(dir_name, file_name)
def main(): parser = argparse.ArgumentParser() # Required parameters parser.add_argument( "--train_data_file", default=None, type=str, required=True, help="The input training data file (a text file)." ) parser.add_argument( "--output_dir", type=str, required=True, help="The output directory where the model predictions and checkpoints will be written.", ) parser.add_argument( "--model_type", type=str, required=True, help="The model architecture to be trained or fine-tuned.", ) # Other parameters parser.add_argument( "--eval_data_file", default=None, type=str, help="An optional input evaluation data file to evaluate the perplexity on (a text file).", ) parser.add_argument( "--line_by_line", action="store_true", help="Whether distinct lines of text in the dataset are to be handled as distinct sequences.", ) parser.add_argument( "--should_continue", action="store_true", help="Whether to continue from latest checkpoint in output_dir" ) parser.add_argument( "--model_name_or_path", default=None, type=str, help="The model checkpoint for weights initialization. Leave None if you want to train a model from scratch.", ) parser.add_argument( "--mlm", action="store_true", help="Train with masked-language modeling loss instead of language modeling." ) parser.add_argument( "--mlm_probability", type=float, default=0.15, help="Ratio of tokens to mask for masked language modeling loss" ) parser.add_argument( "--config_name", default=None, type=str, help="Optional pretrained config name or path if not the same as model_name_or_path. If both are None, initialize a new config.", ) parser.add_argument( "--tokenizer_name", default=None, type=str, help="Optional pretrained tokenizer name or path if not the same as model_name_or_path. If both are None, initialize a new tokenizer.", ) parser.add_argument( "--cache_dir", default=None, type=str, help="Optional directory to store the pre-trained models downloaded from s3 (instead of the default one)", ) parser.add_argument( "--block_size", default=-1, type=int, help="Optional input sequence length after tokenization." "The training dataset will be truncated in block of this size for training." "Default to the model max input length for single sentence inputs (take into account special tokens).", ) parser.add_argument("--do_train", action="store_true", help="Whether to run training.") parser.add_argument("--do_eval", action="store_true", help="Whether to run eval on the dev set.") parser.add_argument( "--evaluate_during_training", action="store_true", help="Run evaluation during training at each logging step." ) parser.add_argument("--per_gpu_train_batch_size", default=4, type=int, help="Batch size per GPU/CPU for training.") parser.add_argument( "--per_gpu_eval_batch_size", default=4, type=int, help="Batch size per GPU/CPU for evaluation." ) parser.add_argument( "--gradient_accumulation_steps", type=int, default=1, help="Number of updates steps to accumulate before performing a backward/update pass.", ) parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.") parser.add_argument("--weight_decay", default=0.0, type=float, help="Weight decay if we apply some.") parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.") parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") parser.add_argument( "--num_train_epochs", default=1.0, type=float, help="Total number of training epochs to perform." ) parser.add_argument( "--max_steps", default=-1, type=int, help="If > 0: set total number of training steps to perform. Override num_train_epochs.", ) # parser.add_argument("--warmup_steps", default=0, type=int, help="Linear warmup over warmup_steps.") parser.add_argument("--pct_warmup", default=0.3, type=float, help="Linear warmup over pct_warmup * total_steps.") parser.add_argument("--logging_steps", type=int, default=500, help="Log every X updates steps.") parser.add_argument("--save_steps", type=int, default=500, help="Save checkpoint every X updates steps.") parser.add_argument( "--save_total_limit", type=int, default=None, help="Limit the total amount of checkpoints, delete the older checkpoints in the output_dir, does not delete by default", ) parser.add_argument( "--eval_all_checkpoints", action="store_true", help="Evaluate all checkpoints starting with the same prefix as model_name_or_path ending and ending with step number", ) parser.add_argument("--no_cuda", action="store_true", help="Avoid using CUDA when available") parser.add_argument( "--overwrite_output_dir", action="store_true", help="Overwrite the content of the output directory" ) parser.add_argument( "--overwrite_cache", action="store_true", help="Overwrite the cached training and evaluation sets" ) parser.add_argument("--seed", type=int, default=42, help="random seed for initialization") parser.add_argument( "--fp16", action="store_true", help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit", ) parser.add_argument( "--fp16_opt_level", type=str, default="O1", help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']." "See details at https://nvidia.github.io/apex/amp.html", ) parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") parser.add_argument("--server_ip", type=str, default="", help="For distant debugging.") parser.add_argument("--server_port", type=str, default="", help="For distant debugging.") args = parser.parse_args() if args.model_type in ["bert", "roberta", "distilbert", "camembert"] and not args.mlm: raise ValueError( "BERT and RoBERTa-like models do not have LM heads but masked LM heads. They must be run using the --mlm " "flag (masked language modeling)." ) if args.eval_data_file is None and args.do_eval: raise ValueError( "Cannot do evaluation without an evaluation data file. Either supply a file to --eval_data_file " "or remove the --do_eval argument." ) if args.should_continue: sorted_checkpoints = _sorted_checkpoints(args) if len(sorted_checkpoints) == 0: raise ValueError("Used --should_continue but no checkpoint was found in --output_dir.") else: args.model_name_or_path = sorted_checkpoints[-1] if ( os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train and not args.overwrite_output_dir ): raise ValueError( "Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format( args.output_dir ) ) # Setup distant debugging if needed if args.server_ip and args.server_port: # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script import ptvsd print("Waiting for debugger attach") ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True) ptvsd.wait_for_attach() # Setup CUDA, GPU & distributed training if args.local_rank == -1 or args.no_cuda: device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") args.n_gpu = 0 if args.no_cuda else torch.cuda.device_count() else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs torch.cuda.set_device(args.local_rank) device = torch.device("cuda", args.local_rank) torch.distributed.init_process_group(backend="nccl") args.n_gpu = 1 args.device = device # Setup logging logging.basicConfig( format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN, ) logger.warning( "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s", args.local_rank, device, args.n_gpu, bool(args.local_rank != -1), args.fp16, ) # Set seed set_seed(args) # Load pretrained model and tokenizer if args.local_rank not in [-1, 0]: torch.distributed.barrier() # Barrier to make sure only the first process in distributed training download model & vocab config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type] if args.config_name: config = config_class.from_pretrained(args.config_name, cache_dir=args.cache_dir) elif args.model_name_or_path: config = config_class.from_pretrained(args.model_name_or_path, cache_dir=args.cache_dir) else: config = config_class() if args.tokenizer_name: # tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name, cache_dir=args.cache_dir) tokenizer = ByteLevelBPETokenizer( os.path.join(args.tokenizer_name, "vocab.json"), os.path.join(args.tokenizer_name, "merges.txt") ) tokenizer._tokenizer.post_processor = BertProcessing( ("</s>", tokenizer.token_to_id("</s>")), ("<s>", tokenizer.token_to_id("<s>")), ) tokenizer.enable_truncation(max_length=512) elif args.model_name_or_path: tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path, cache_dir=args.cache_dir) else: raise ValueError( "You are instantiating a new {} tokenizer. This is not supported, but you can do it from another script, save it," "and load it from here, using --tokenizer_name".format(tokenizer_class.__name__) ) # if args.block_size <= 0: # args.block_size = tokenizer.max_len # # Our input block size will be the max possible for the model # else: # args.block_size = min(args.block_size, tokenizer.max_len) if args.model_name_or_path: model = model_class.from_pretrained( args.model_name_or_path, from_tf=bool(".ckpt" in args.model_name_or_path), config=config, cache_dir=args.cache_dir, ) else: logger.info("Training new model from scratch") model = model_class(config=config) model.to(args.device) if args.local_rank == 0: torch.distributed.barrier() # End of barrier to make sure only the first process in distributed training download model & vocab logger.info("Training/evaluation parameters %s", args) # Training if args.do_train: if args.local_rank not in [-1, 0]: torch.distributed.barrier() # Barrier to make sure only the first process in distributed training process the dataset, and the others will use the cache train_dataset = load_and_cache_examples(args, tokenizer, evaluate=False) if args.local_rank == 0: torch.distributed.barrier() global_step, tr_loss = train(args, train_dataset, model, tokenizer) logger.info(" global_step = %s, average loss = %s", global_step, tr_loss) # Saving best-practices: if you use save_pretrained for the model and tokenizer, you can reload them using from_pretrained() if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0): # Create output directory if needed if args.local_rank in [-1, 0]: os.makedirs(args.output_dir, exist_ok=True) logger.info("Saving model checkpoint to %s", args.output_dir) # Save a trained model, configuration and tokenizer using `save_pretrained()`. # They can then be reloaded using `from_pretrained()` model_to_save = ( model.module if hasattr(model, "module") else model ) # Take care of distributed/parallel training model_to_save.save_pretrained(args.output_dir) tokenizer.save_pretrained(args.output_dir) # Good practice: save your training arguments together with the trained model torch.save(args, os.path.join(args.output_dir, "training_args.bin")) # Load a trained model and vocabulary that you have fine-tuned model = model_class.from_pretrained(args.output_dir) tokenizer = tokenizer_class.from_pretrained(args.output_dir) model.to(args.device) # Evaluation results = {} if args.do_eval and args.local_rank in [-1, 0]: checkpoints = [args.output_dir] if args.eval_all_checkpoints: checkpoints = list( os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + "/**/" + WEIGHTS_NAME, recursive=True)) ) logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN) # Reduce logging logger.info("Evaluate the following checkpoints: %s", checkpoints) for checkpoint in checkpoints: global_step = checkpoint.split("-")[-1] if len(checkpoints) > 1 else "" prefix = checkpoint.split("/")[-1] if checkpoint.find("checkpoint") != -1 else "" model = model_class.from_pretrained(checkpoint) model.to(args.device) result = evaluate(args, model, tokenizer, prefix=prefix) result = dict((k + "_{}".format(global_step), v) for k, v in result.items()) results.update(result) return results
def __init__(self, cfg): super().__init__(cfg) self.scales = [str((cfg.load_size // (2**i))) for i in range(3)] self.scales.reverse() self.device_map = { 'style': self.devices[0], 'content': self.devices[0], 'img': self.devices[0] } self.network_names = [ 'style_model', 'content_model', 'generator', 'discriminators' ] self.device_name_map = { 'style_model': 'style', 'content_model': 'content', 'generators': 'img', 'discriminators': 'img' } tokenizer = ByteLevelBPETokenizer( "vocab.json", "merges.txt", ) tokenizer._tokenizer.post_processor = BertProcessing( ("</s>", tokenizer.token_to_id("</s>")), ("<s>", tokenizer.token_to_id("<s>")), ) self.style_l_model = LanguageModel(cfg, tokenizer, self.device_map['style']).to( self.device_map['style']) self.style_f_model = VAE(cfg.rnn_hidden_dim * 2, self.device_map['style'], cfg).to(self.device_map['style']) self.content_l_model = LanguageModel(cfg, tokenizer, self.device_map['content']).to( self.device_map['content']) self.content_f_model = VAE(cfg.rnn_hidden_dim * 2, self.device_map['content'], cfg).to(self.device_map['content']) self.generator = StyleGenerator(cfg).to(self.device_map['img']) self.discriminators = {} for scale in self.scales: self.discriminators[scale] = PatchConvolutionalDiscriminator( int(scale), self.device_map['img'], cfg) self.discriminators = nn.ModuleDict(self.discriminators).to( self.device_map['img']) self.visual_names = ['visual_dict'] self.visual_dict = {'real': None, 'fake': None} self.loss_names = ['loss'] self.visualizer = Visualizer(cfg) self.image_pool = {scale: ImagePool(cfg) for scale in self.scales} self.consistency_criterion = ColorConsistencyLoss(cfg).to( self.device_map['img']) self.distribution_criterion = KLDLoss().to(self.device_map['img']) self.generator_criterion = BinaryCrossEntropyLoss(cfg).to( self.device_map['img'])
class BERTokenizer: ''' The basic element of preprocessing would be a pretrained tokenizer. This is a wrapper on top of such tokenizers to support preprocessing with spacy/stanza. Currently support: BERT, Electra TODO: Robera has not been support yet because its way of hanlding space ''' sp_nlp = None def __init__(self, version): # download vocab files cache = os.path.join(os.environ.get("CACHE_DIR", os.getcwd()), ".vector_cache") vocab_dir = os.path.join(cache, f"{version}") if not os.path.exists(vocab_dir): pretrained_tokenizer = AutoTokenizer.from_pretrained(version) pretrained_tokenizer.save_pretrained(vocab_dir) if "uncased" in version or "cased" not in version: lowercase = True # roberta, electra, bert-base-uncased else: lowercase = False # bert-cased if version.startswith("bert") or "electra" in version: vocab_path = os.path.join(vocab_dir, "vocab.txt") self.tokenizer = BertWordPieceTokenizer(vocab_path, lowercase=lowercase) elif version.startswith("roberta"): vocab_path = os.path.join(vocab_dir, "vocab.json") merge_path = os.path.join(vocab_dir, "merges.txt") self.tokenizer = ByteLevelBPETokenizer(vocab_path, merge_path, lowercase=lowercase) else: raise NotImplementedError self.cls_token = self.tokenizer._parameters["cls_token"] self.cls_token_id = self.tokenizer.token_to_id(self.cls_token) self.sep_token = self.tokenizer._parameters["sep_token"] self.sep_token_id = self.tokenizer.token_to_id(self.sep_token) self.pad_token = self.tokenizer._parameters["pad_token"] self.pad_token_id = self.tokenizer.token_to_id(self.pad_token) def _encode(self, input_): if isinstance(input_, list) or isinstance(input_, tuple): encodes = self.tokenizer.encode(input_, is_pretokenized=True) else: encodes = self.tokenizer.encode(input_) return encodes def tokenize(self, text): encodes = self._encode(text) tokens = encodes.tokens[1:-1] return tokens def tokenize_and_lemmatize(self, text, lang="en"): """ This will be used for matching 1) remove cls and sep 2) lemmatize """ if BERTokenizer.sp_nlp is None: snlp = stanza.Pipeline(lang=lang, use_gpu=True, tokenize_pretokenized=True) BERTokenizer.sp_nlp = StanzaLanguage(snlp) encodes = self._encode(text) tokens = encodes.tokens[1:-1] norm_tokens = [t.lemma_ for t in self.sp_nlp([tokens])] return norm_tokens def tokenize_with_orig(self, text): """ Tokenize but return the original chars, this would be helpful for copying operations. """ # TODO: if text is a list, change accordingly how the offset is computed assert isinstance(text, str) encodes = self._encode(text) orig_tokens = [text[i:j] for i, j in encodes.offsets[1:-1]] return orig_tokens def tokenize_and_spacy(self, text, lang="en"): """ Keep meta information from spacy, used for matching """ if BERTokenizer.sp_nlp is None: snlp = stanza.Pipeline(lang=lang, use_gpu=True, tokenize_pretokenized=True) BERTokenizer.sp_nlp = StanzaLanguage(snlp) tokens = self.tokenizer.encode(text).tokens[1:-1] return self.sp_nlp([tokens]) def check_bert_input_seq(self, toks): if toks[0] == self.cls_token_id and toks[-1] == self.sep_token_id: return True else: return False def pieces_to_words(self, pieces): """ TODO: use general variable of prefix """ words = [] cur_word = None for piece in pieces: if piece.startswith("##"): cur_word = cur_word + piece[2:] else: if cur_word is not None: words.append(cur_word) cur_word = piece return words def text_to_ids(self, sent, cls=True): """ This function is primarily used convert text to bpe token ids """ encs = self._encode(sent) if cls: return encs.ids else: assert encs.tokens[0] == self.cls_token return encs.ids[1:] # remove CLS def pad_sequence_for_bert_batch(self, tokens_lists): """ 1) Pad with pad token 2) Generate token_type_list """ pad_id = self.pad_token_id max_len = max([len(it) for it in tokens_lists]) assert max_len <= 512 toks_ids = [] att_masks = [] tok_type_lists = [] for item_toks in tokens_lists: padded_item_toks = item_toks + [pad_id ] * (max_len - len(item_toks)) toks_ids.append(padded_item_toks) _att_mask = [1] * len(item_toks) + [0] * (max_len - len(item_toks)) att_masks.append(_att_mask) first_sep_id = padded_item_toks.index(self.sep_token_id) assert first_sep_id > 0 _tok_type_list = [0] * (first_sep_id + 1) + [1] * (max_len - first_sep_id - 1) tok_type_lists.append(_tok_type_list) return toks_ids, att_masks, tok_type_lists
class Tokenizer: def __init__(self, model_name, vocab_file, *, merges_file=None, lowercase=True, handle_chinese_chars=False, dropout=None): self.model_name = model_name if model_name == 'bert': self._pad_token = '[PAD]' self._sep_token = '[SEP]' self._cls_token = '[CLS]' self._unk_token = '[UNK]' if dropout is not None: logger.warning( 'BPE dropout is not supported by BertWordPieceTokenizer.') self.tokenizer = BertWordPieceTokenizer( vocab_file, lowercase=lowercase, handle_chinese_chars=handle_chinese_chars, unk_token=self.unk_token, cls_token=self.cls_token, sep_token=self.sep_token) elif model_name == 'roberta': if merges_file is None: raise AttributeError( 'To use ByteLevelTokenizer, specify path to merges file.') self._pad_token = '<pad>' self._sep_token = '</s>' self._cls_token = '<s>' self._unk_token = '<unk>' try: self.tokenizer = ByteLevelBPETokenizer(vocab_file=vocab_file, merges_file=merges_file, dropout=dropout) except TypeError as e: logger.warning( 'BPE dropout is not supported by ByteLevelBPETokenizer.') logger.error(e) self.tokenizer = ByteLevelBPETokenizer(vocab_file=vocab_file, merges_file=merges_file) else: raise NotImplementedError( f'Tokenizer initialization for model {model_name} is not implemented.' ) def __len__(self): return self.tokenizer._tokenizer.get_vocab_size() def encode(self, string): return self.tokenizer.encode(string).ids def decode(self, ids, *, skip_special_tokens=True): return self.tokenizer.decode( ids, skip_special_tokens=skip_special_tokens).replace(' ##', '') @property def pad_token_id(self): return self.tokenizer.token_to_id(self._pad_token) @property def sep_token_id(self): return self.tokenizer.token_to_id(self._sep_token) @property def cls_token_id(self): return self.tokenizer.token_to_id(self._cls_token) @property def unk_token_id(self): return self.tokenizer.token_to_id(self._unk_token) @property def pad_token(self): return self._pad_token @property def sep_token(self): return self._sep_token @property def cls_token(self): return self._cls_token @property def unk_token(self): return self._unk_token
class HuggingfaceTokenizerBPE(nn.Module): def __init__(self, text_files, dataset_info_path='', config_data=None): super().__init__() # The default vocab size in the BERT model is 30522. If we want a number larger than that, we will also have to # change the BERT configuration. vocab_size = 30000 self.info = f'hug{vocab_size}' with open(f'config/data/{config_data}.json') as json_file: tokenizer_from = json.load(json_file)['tokenizer_from'] config_name = config_data if tokenizer_from == "" else tokenizer_from print( os.path.join(dataset_info_path, f'tokenizer_{config_name}_{vocab_size}-vocab.json')) # The loading is only properly implemented starting from version 0.8. However, it makes the system use a lot of # CPU for no reason (it is much slower). Maybe it will be fixed in the future. if not os.path.isfile( os.path.join( dataset_info_path, f'tokenizer_{config_name}_{vocab_size}-vocab.json')): text_files = text_files() self.tokenizer = ByteLevelBPETokenizer() # Join into a single file. This should NOT be necessary but it does not work properly with a lot of files with open('/tmp/text_files.txt', 'wb') as outfile: for filename in tqdm( text_files, desc='Joining all files into one for tokenization'): with open(filename, 'rb') as readfile: shutil.copyfileobj(readfile, outfile) text_files = '/tmp/text_files.txt' self.tokenizer.train(text_files, vocab_size=vocab_size, special_tokens=special_tokens) self.tokenizer.save(dataset_info_path, f'tokenizer_{config_name}_{vocab_size}') # No "else", always load for consistency vocab_file = os.path.join( dataset_info_path, f'tokenizer_{config_name}_{vocab_size}-vocab.json') merges_file = os.path.join( dataset_info_path, f'tokenizer_{config_name}_{vocab_size}-merges.txt') self.tokenizer = ByteLevelBPETokenizer(vocab_file=vocab_file, merges_file=merges_file) self.tokenizer.add_special_tokens(special_tokens) self.index_special_tokens = { tok: self.tokenizer.encode(tok).ids[0] for tok in special_tokens } @property def device(self): return self._float_tensor.device def encode(self, sentence: str): output = self.tokenizer.encode(sentence) token_ids = output.ids tokens = output.tokens return torch.tensor(token_ids), tokens def decode(self, tokens: torch.LongTensor): assert tokens.dim() == 1 tokens = list(tokens.cpu().numpy()) sentences = self.tokenizer.decode(tokens) return sentences def id_to_token(self, token_id): if type(token_id) != torch.Tensor: token_id = torch.tensor(token_id) return self.tokenizer.id_to_token(token_id) def token_to_id(self, token): assert type(token) == str return self.tokenizer.token_to_id(token) def __len__(self): return self.tokenizer.get_vocab_size() # This is simply for PyCharm to find the correct reference to the methods of the class def __call__(self, *input, **kwargs) -> typing.Any: return super().__call__(*input, **kwargs)