Python load_loose_json Exemples

Exemple #1

 def load_mlm_data(path):
     from pytorch_pretrained_bert.tokenization import BertTokenizer
     tokenizer = BertTokenizer.from_pretrained(bert_model,
                                               do_lower_case=do_lower_case)
     vocab_words = list(tokenizer.vocab.keys())
     data = load_loose_json(path)
     docs = []
     for doc in data:
         paras = doc['text'].split('\n\n')
         paras = [para.strip() for para in paras if len(para.strip()) > 0]
         tokens = [tokenizer.tokenize(para) for para in paras]
         docs.append(tokens)
     return docs, tokenizer

Exemple #2

Fichier : batcher.py Projet : namisan/mt-dnn

 def load_mlm_data(path):
     from transformers import AutoTokenizer
     tokenizer = AutoTokenizer.from_pretrained(bert_model,
                                               cache_dir=".cache")
     vocab_words = list(tokenizer.vocab.keys())
     data = load_loose_json(path)
     docs = []
     for doc in data:
         paras = doc["text"].split("\n\n")
         paras = [
             para.strip() for para in paras if len(para.strip()) > 0
         ]
         tokens = [tokenizer.tokenize(para) for para in paras]
         docs.append(tokens)
     return docs, tokenizer