def test_dpr_training(): batch_size = 1 n_epochs = 1 distributed = False # enable for multi GPU training via DDP evaluate_every = 1 question_lang_model = "microsoft/MiniLM-L12-H384-uncased" passage_lang_model = "microsoft/MiniLM-L12-H384-uncased" do_lower_case = True use_fast = True similarity_function = "dot_product" device, n_gpu = initialize_device_settings(use_cuda=False) query_tokenizer = Tokenizer.load( pretrained_model_name_or_path=question_lang_model, do_lower_case=do_lower_case, use_fast=use_fast) passage_tokenizer = Tokenizer.load( pretrained_model_name_or_path=passage_lang_model, do_lower_case=do_lower_case, use_fast=use_fast) label_list = ["hard_negative", "positive"] processor = TextSimilarityProcessor(query_tokenizer=query_tokenizer, passage_tokenizer=passage_tokenizer, max_seq_len_query=10, max_seq_len_passage=10, label_list=label_list, metric="text_similarity_metric", data_dir="samples/dpr/", train_filename="sample.json", dev_filename="sample.json", test_filename=None, embed_title=True, num_hard_negatives=1, dev_split=0, max_samples=2) data_silo = DataSilo(processor=processor, batch_size=batch_size, distributed=False) question_language_model = LanguageModel.load( pretrained_model_name_or_path=question_lang_model, language_model_class="DPRQuestionEncoder") passage_language_model = LanguageModel.load( pretrained_model_name_or_path=passage_lang_model, language_model_class="DPRContextEncoder") prediction_head = TextSimilarityHead( similarity_function=similarity_function) model = BiAdaptiveModel( language_model1=question_language_model, language_model2=passage_language_model, prediction_heads=[prediction_head], embeds_dropout_prob=0.1, lm1_output_types=["per_sequence"], lm2_output_types=["per_sequence"], device=device, ) model, optimizer, lr_schedule = initialize_optimizer( model=model, learning_rate=1e-5, optimizer_opts={"name": "TransformersAdamW", "correct_bias": True, "weight_decay": 0.0, \ "eps": 1e-08}, schedule_opts={"name": "LinearWarmup", "num_warmup_steps": 100}, n_batches=len(data_silo.loaders["train"]), n_epochs=n_epochs, grad_acc_steps=1, device=device, distributed=distributed ) trainer = Trainer( model=model, optimizer=optimizer, data_silo=data_silo, epochs=n_epochs, n_gpu=n_gpu, lr_schedule=lr_schedule, evaluate_every=evaluate_every, device=device, ) trainer.train() ######## save and load model again save_dir = Path("testsave/dpr-model") model.save(save_dir) del model model2 = BiAdaptiveModel.load(save_dir, device=device) model2, optimizer2, lr_schedule = initialize_optimizer( model=model2, learning_rate=1e-5, optimizer_opts={"name": "TransformersAdamW", "correct_bias": True, "weight_decay": 0.0, \ "eps": 1e-08}, schedule_opts={"name": "LinearWarmup", "num_warmup_steps": 100}, n_batches=len(data_silo.loaders["train"]), n_epochs=n_epochs, grad_acc_steps=1, device=device, distributed=distributed ) trainer2 = Trainer( model=model2, optimizer=optimizer, data_silo=data_silo, epochs=n_epochs, n_gpu=n_gpu, lr_schedule=lr_schedule, evaluate_every=evaluate_every, device=device, ) trainer2.train()
def test_dpr_modules(caplog=None): if caplog: caplog.set_level(logging.CRITICAL) set_all_seeds(seed=42) device, n_gpu = initialize_device_settings(use_cuda=True) # 1.Create question and passage tokenizers query_tokenizer = Tokenizer.load( pretrained_model_name_or_path= "facebook/dpr-question_encoder-single-nq-base", do_lower_case=True, use_fast=True) passage_tokenizer = Tokenizer.load( pretrained_model_name_or_path="facebook/dpr-ctx_encoder-single-nq-base", do_lower_case=True, use_fast=True) processor = TextSimilarityProcessor( query_tokenizer=query_tokenizer, passage_tokenizer=passage_tokenizer, max_seq_len_query=256, max_seq_len_passage=256, label_list=["hard_negative", "positive"], metric="text_similarity_metric", data_dir="data/retriever", train_filename="nq-train.json", dev_filename="nq-dev.json", test_filename="nq-dev.json", embed_title=True, num_hard_negatives=1) question_language_model = LanguageModel.load( pretrained_model_name_or_path="bert-base-uncased", language_model_class="DPRQuestionEncoder", hidden_dropout_prob=0, attention_probs_dropout_prob=0) passage_language_model = LanguageModel.load( pretrained_model_name_or_path="bert-base-uncased", language_model_class="DPRContextEncoder", hidden_dropout_prob=0, attention_probs_dropout_prob=0) prediction_head = TextSimilarityHead(similarity_function="dot_product") model = BiAdaptiveModel( language_model1=question_language_model, language_model2=passage_language_model, prediction_heads=[prediction_head], embeds_dropout_prob=0.0, lm1_output_types=["per_sequence"], lm2_output_types=["per_sequence"], device=device, ) model.connect_heads_with_processor(processor.tasks) assert type(model) == BiAdaptiveModel assert type(processor) == TextSimilarityProcessor assert type(question_language_model) == DPRQuestionEncoder assert type(passage_language_model) == DPRContextEncoder # check embedding layer weights assert list(model.named_parameters())[0][1][ 0, 0].item() - -0.010200000368058681 < 0.0001 d = { 'query': 'big little lies season 2 how many episodes', 'passages': [{ 'title': 'Big Little Lies (TV series)', 'text': 'series garnered several accolades. It received 16 Emmy Award nominations and won eight, including Outstanding Limited Series and acting awards for Kidman, Skarsgård, and Dern. The trio also won Golden Globe Awards in addition to a Golden Globe Award for Best Miniseries or Television Film win for the series. Kidman and Skarsgård also received Screen Actors Guild Awards for their performances. Despite originally being billed as a miniseries, HBO renewed the series for a second season. Production on the second season began in March 2018 and is set to premiere in 2019. All seven episodes are being written by Kelley', 'label': 'positive', 'external_id': '18768923' }, { 'title': 'Little People, Big World', 'text': 'final minutes of the season two-A finale, "Farm Overload". A crowd had gathered around Jacob, who was lying on the ground near the trebuchet. The first two episodes of season two-B focus on the accident, and how the local media reacted to it. The first season of "Little People, Big World" generated solid ratings for TLC (especially in the important 18–49 demographic), leading to the show\'s renewal for a second season. Critical reviews of the series have been generally positive, citing the show\'s positive portrayal of little people. Conversely, other reviews have claimed that the show has a voyeuristic bend', 'label': 'hard_negative', 'external_id': '7459116' }, { 'title': 'Cormac McCarthy', 'text': 'chores of the house, Lee was asked by Cormac to also get a day job so he could focus on his novel writing. Dismayed with the situation, she moved to Wyoming, where she filed for divorce and landed her first job teaching. Cormac McCarthy is fluent in Spanish and lived in Ibiza, Spain, in the 1960s and later settled in El Paso, Texas, where he lived for nearly 20 years. In an interview with Richard B. Woodward from "The New York Times", "McCarthy doesn\'t drink anymore – he quit 16 years ago in El Paso, with one of his young', 'label': 'negative', 'passage_id': '2145653' }] } dataset, tensor_names, _ = processor.dataset_from_dicts( dicts=[d], return_baskets=False) features = { key: val.unsqueeze(0).to(device) for key, val in zip(tensor_names, dataset[0]) } # test features assert torch.all( torch.eq( features["query_input_ids"][0][:10].cpu(), torch.tensor( [101, 2502, 2210, 3658, 2161, 1016, 2129, 2116, 4178, 102]))) assert torch.all( torch.eq( features["passage_input_ids"][0][0][:10].cpu(), torch.tensor( [101, 2502, 2210, 3658, 1006, 2694, 2186, 1007, 102, 2186]))) assert len(features["query_segment_ids"][0].nonzero()) == 0 assert len(features["passage_segment_ids"][0].nonzero()) == 0 assert torch.all( torch.eq(features["query_attention_mask"].nonzero()[:, 1].cpu(), torch.tensor(list(range(10))))) assert torch.all( torch.eq( features["passage_attention_mask"][0][0].nonzero().cpu().squeeze(), torch.tensor(list(range(127))))) assert torch.all( torch.eq( features["passage_attention_mask"][0][1].nonzero().cpu().squeeze(), torch.tensor(list(range(143))))) # test model encodings query_vector = model.language_model1(**features)[0] passage_vector = model.language_model2(**features)[0] assert torch.all( torch.le( query_vector[0, :10].cpu() - torch.tensor([ -0.2135, -0.4748, 0.0501, -0.0430, -0.1747, -0.0441, 0.5638, 0.1405, 0.2285, 0.0893 ]), torch.ones((1, 10)) * 0.0001)) assert torch.all( torch.le( passage_vector[0, :10].cpu() - torch.tensor([ 0.0557, -0.6836, -0.3645, -0.5566, 0.2034, -0.3656, 0.2969, -0.0555, 0.3405, -0.8691 ]), torch.ones((1, 10)) * 0.0001)) assert torch.all( torch.le( passage_vector[1, :10].cpu() - torch.tensor([ -0.2006, -1.5002, -0.1897, -0.3421, -0.0405, -0.0471, -0.0306, 0.1156, 0.3350, -0.3412 ]), torch.ones((1, 10)) * 0.0001)) # test logits and loss embeddings = model(**features) query_emb, passage_emb = embeddings[0] assert torch.all(torch.eq(query_emb.cpu(), query_vector.cpu())) assert torch.all(torch.eq(passage_emb.cpu(), passage_vector.cpu())) loss = model.logits_to_loss_per_head(embeddings, **features) similarity_scores = model.prediction_heads[0]._embeddings_to_scores( query_emb, passage_emb).cpu() assert torch.all( torch.le( similarity_scores - torch.tensor([[-1.8311e-03, -6.3016e+00]]), torch.ones((1, 2)) * 0.0001)) assert (loss[0].item() - 0.0018) <= 0.0001
def __init__(self, document_store: BaseDocumentStore, query_embedding_model: Union[ Path, str] = "facebook/dpr-question_encoder-single-nq-base", passage_embedding_model: Union[ Path, str] = "facebook/dpr-ctx_encoder-single-nq-base", model_version: Optional[str] = None, max_seq_len_query: int = 64, max_seq_len_passage: int = 256, use_gpu: bool = True, batch_size: int = 16, embed_title: bool = True, use_fast_tokenizers: bool = True, infer_tokenizer_classes: bool = False, similarity_function: str = "dot_product", progress_bar: bool = True): """ Init the Retriever incl. the two encoder models from a local or remote model checkpoint. The checkpoint format matches huggingface transformers' model format **Example:** ```python | # remote model from FAIR | DensePassageRetriever(document_store=your_doc_store, | query_embedding_model="facebook/dpr-question_encoder-single-nq-base", | passage_embedding_model="facebook/dpr-ctx_encoder-single-nq-base") | # or from local path | DensePassageRetriever(document_store=your_doc_store, | query_embedding_model="model_directory/question-encoder", | passage_embedding_model="model_directory/context-encoder") ``` :param document_store: An instance of DocumentStore from which to retrieve documents. :param query_embedding_model: Local path or remote name of question encoder checkpoint. The format equals the one used by hugging-face transformers' modelhub models Currently available remote names: ``"facebook/dpr-question_encoder-single-nq-base"`` :param passage_embedding_model: Local path or remote name of passage encoder checkpoint. The format equals the one used by hugging-face transformers' modelhub models Currently available remote names: ``"facebook/dpr-ctx_encoder-single-nq-base"`` :param model_version: The version of model to use from the HuggingFace model hub. Can be tag name, branch name, or commit hash. :param max_seq_len_query: Longest length of each query sequence. Maximum number of tokens for the query text. Longer ones will be cut down." :param max_seq_len_passage: Longest length of each passage/context sequence. Maximum number of tokens for the passage text. Longer ones will be cut down." :param use_gpu: Whether to use gpu or not :param batch_size: Number of questions or passages to encode at once :param embed_title: Whether to concatenate title and passage to a text pair that is then used to create the embedding. This is the approach used in the original paper and is likely to improve performance if your titles contain meaningful information for retrieval (topic, entities etc.) . The title is expected to be present in doc.meta["name"] and can be supplied in the documents before writing them to the DocumentStore like this: {"text": "my text", "meta": {"name": "my title"}}. :param use_fast_tokenizers: Whether to use fast Rust tokenizers :param infer_tokenizer_classes: Whether to infer tokenizer class from the model config / name. If `False`, the class always loads `DPRQuestionEncoderTokenizer` and `DPRContextEncoderTokenizer`. :param similarity_function: Which function to apply for calculating the similarity of query and passage embeddings during training. Options: `dot_product` (Default) or `cosine` :param progress_bar: Whether to show a tqdm progress bar or not. Can be helpful to disable in production deployments to keep the logs clean. """ self.document_store = document_store self.batch_size = batch_size self.max_seq_len_passage = max_seq_len_passage self.max_seq_len_query = max_seq_len_query self.progress_bar = progress_bar if document_store is None: logger.warning( "DensePassageRetriever initialized without a document store. " "This is fine if you are performing DPR training. " "Otherwise, please provide a document store in the constructor." ) elif document_store.similarity != "dot_product": logger.warning( f"You are using a Dense Passage Retriever model with the {document_store.similarity} function. " "We recommend you use dot_product instead. " "This can be set when initializing the DocumentStore") if use_gpu and torch.cuda.is_available(): self.device = torch.device("cuda") else: self.device = torch.device("cpu") self.embed_title = embed_title self.infer_tokenizer_classes = infer_tokenizer_classes tokenizers_default_classes = { "query": "DPRQuestionEncoderTokenizer", "passage": "DPRContextEncoderTokenizer" } if self.infer_tokenizer_classes: tokenizers_default_classes["query"] = None # type: ignore tokenizers_default_classes["passage"] = None # type: ignore # Init & Load Encoders self.query_tokenizer = Tokenizer.load( pretrained_model_name_or_path=query_embedding_model, revision=model_version, do_lower_case=True, use_fast=use_fast_tokenizers, tokenizer_class=tokenizers_default_classes["query"]) self.query_encoder = LanguageModel.load( pretrained_model_name_or_path=query_embedding_model, revision=model_version, language_model_class="DPRQuestionEncoder") self.passage_tokenizer = Tokenizer.load( pretrained_model_name_or_path=passage_embedding_model, revision=model_version, do_lower_case=True, use_fast=use_fast_tokenizers, tokenizer_class=tokenizers_default_classes["passage"]) self.passage_encoder = LanguageModel.load( pretrained_model_name_or_path=passage_embedding_model, revision=model_version, language_model_class="DPRContextEncoder") self.processor = TextSimilarityProcessor( tokenizer=self.query_tokenizer, passage_tokenizer=self.passage_tokenizer, max_seq_len_passage=self.max_seq_len_passage, max_seq_len_query=self.max_seq_len_query, label_list=["hard_negative", "positive"], metric="text_similarity_metric", embed_title=self.embed_title, num_hard_negatives=0, num_positives=1) prediction_head = TextSimilarityHead( similarity_function=similarity_function) self.model = BiAdaptiveModel( language_model1=self.query_encoder, language_model2=self.passage_encoder, prediction_heads=[prediction_head], embeds_dropout_prob=0.1, lm1_output_types=["per_sequence"], lm2_output_types=["per_sequence"], device=self.device, ) self.model.connect_heads_with_processor(self.processor.tasks, require_labels=False)
def dense_passage_retrieval(): logging.basicConfig( format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", datefmt="%m/%d/%Y %H:%M:%S", level=logging.INFO, ) ml_logger = MLFlowLogger(tracking_uri="https://public-mlflow.deepset.ai/") ml_logger.init_experiment(experiment_name="FARM-dense_passage_retrieval", run_name="Run_dpr_enocder") ########################## ########## Settings ########################## set_all_seeds(seed=42) device, n_gpu = initialize_device_settings(use_cuda=True) batch_size = 2 n_epochs = 3 evaluate_every = 1000 question_lang_model = "facebook/dpr-question_encoder-single-nq-base" passage_lang_model = "facebook/dpr-ctx_encoder-single-nq-base" do_lower_case = True use_fast = True embed_title = True num_hard_negatives = 1 similarity_function = "dot_product" train_filename = "nq-train.json" dev_filename = "nq-dev.json" max_samples = None #load a smaller dataset (e.g. for debugging) # 1.Create question and passage tokenizers query_tokenizer = Tokenizer.load( pretrained_model_name_or_path=question_lang_model, do_lower_case=do_lower_case, use_fast=use_fast) context_tokenizer = Tokenizer.load( pretrained_model_name_or_path=passage_lang_model, do_lower_case=do_lower_case, use_fast=use_fast) # 2. Create a DataProcessor that handles all the conversion from raw text into a pytorch Dataset # data_dir "data/retriever" should contain DPR training and dev files downloaded from https://github.com/facebookresearch/DPR # i.e., nq-train.json, nq-dev.json or trivia-train.json, trivia-dev.json label_list = ["hard_negative", "positive"] metric = "text_similarity_metric" processor = TextSimilarityProcessor(tokenizer=query_tokenizer, passage_tokenizer=context_tokenizer, max_seq_len=256, label_list=label_list, metric=metric, data_dir="data/retriever", train_filename=train_filename, dev_filename=dev_filename, test_filename=dev_filename, embed_title=embed_title, num_hard_negatives=num_hard_negatives, max_samples=max_samples) # 3. Create a DataSilo that loads several datasets (train/dev/test), provides DataLoaders for them and calculates a few descriptive statistics of our datasets # NOTE: In FARM, the dev set metrics differ from test set metrics in that they are calculated on a token level instead of a word level data_silo = DataSilo(processor=processor, batch_size=batch_size, distributed=False) # 4. Create an AdaptiveModel+ # a) which consists of a pretrained language model as a basis question_language_model = LanguageModel.load( pretrained_model_name_or_path="bert-base-uncased", language_model_class="DPRQuestionEncoder") passage_language_model = LanguageModel.load( pretrained_model_name_or_path="bert-base-uncased", language_model_class="DPRContextEncoder") # b) and a prediction head on top that is suited for our task => Question Answering prediction_head = TextSimilarityHead( similarity_function=similarity_function) model = BiAdaptiveModel( language_model1=question_language_model, language_model2=passage_language_model, prediction_heads=[prediction_head], embeds_dropout_prob=0.1, lm1_output_types=["per_sequence"], lm2_output_types=["per_sequence"], device=device, ) # 5. Create an optimizer model, optimizer, lr_schedule = initialize_optimizer( model=model, learning_rate=1e-5, optimizer_opts={"name": "TransformersAdamW", "correct_bias": True, "weight_decay": 0.0, \ "eps": 1e-08}, schedule_opts={"name": "LinearWarmup", "num_warmup_steps": 100}, n_batches=len(data_silo.loaders["train"]), n_epochs=n_epochs, grad_acc_steps=1, device=device ) # 6. Feed everything to the Trainer, which keeps care of growing our model and evaluates it from time to time trainer = Trainer( model=model, optimizer=optimizer, data_silo=data_silo, epochs=n_epochs, n_gpu=n_gpu, lr_schedule=lr_schedule, evaluate_every=evaluate_every, device=device, ) # 7. Let it grow! Watch the tracked metrics live on the public mlflow server: https://public-mlflow.deepset.ai trainer.train() # 8. Hooray! You have a model. Store it: save_dir = Path("../saved_models/dpr-tutorial") model.save(save_dir) processor.save(save_dir) # 9. Evaluate test_data_loader = data_silo.get_data_loader("test") if test_data_loader is not None: evaluator_test = Evaluator(data_loader=test_data_loader, tasks=data_silo.processor.tasks, device=device) model.connect_heads_with_processor(processor.tasks) test_result = evaluator_test.eval(model)
def __init__(self, document_store: BaseDocumentStore, query_embedding_model: Union[ Path, str] = "facebook/dpr-question_encoder-single-nq-base", passage_embedding_model: Union[ Path, str] = "facebook/dpr-ctx_encoder-single-nq-base", max_seq_len_query: int = 64, max_seq_len_passage: int = 256, use_gpu: bool = True, batch_size: int = 16, embed_title: bool = True, use_fast_tokenizers: bool = True, similarity_function: str = "dot_product"): """ Init the Retriever incl. the two encoder models from a local or remote model checkpoint. The checkpoint format matches huggingface transformers' model format **Example:** ```python | # remote model from FAIR | DensePassageRetriever(document_store=your_doc_store, | query_embedding_model="facebook/dpr-question_encoder-single-nq-base", | passage_embedding_model="facebook/dpr-ctx_encoder-single-nq-base") | # or from local path | DensePassageRetriever(document_store=your_doc_store, | query_embedding_model="model_directory/question-encoder", | passage_embedding_model="model_directory/context-encoder") ``` :param document_store: An instance of DocumentStore from which to retrieve documents. :param query_embedding_model: Local path or remote name of question encoder checkpoint. The format equals the one used by hugging-face transformers' modelhub models Currently available remote names: ``"facebook/dpr-question_encoder-single-nq-base"`` :param passage_embedding_model: Local path or remote name of passage encoder checkpoint. The format equals the one used by hugging-face transformers' modelhub models Currently available remote names: ``"facebook/dpr-ctx_encoder-single-nq-base"`` :param max_seq_len_query: Longest length of each query sequence. Maximum number of tokens for the query text. Longer ones will be cut down." :param max_seq_len_passage: Longest length of each passage/context sequence. Maximum number of tokens for the passage text. Longer ones will be cut down." :param use_gpu: Whether to use gpu or not :param batch_size: Number of questions or passages to encode at once :param embed_title: Whether to concatenate title and passage to a text pair that is then used to create the embedding. This is the approach used in the original paper and is likely to improve performance if your titles contain meaningful information for retrieval (topic, entities etc.) . The title is expected to be present in doc.meta["name"] and can be supplied in the documents before writing them to the DocumentStore like this: {"text": "my text", "meta": {"name": "my title"}}. """ self.document_store = document_store self.batch_size = batch_size self.max_seq_len_passage = max_seq_len_passage self.max_seq_len_query = max_seq_len_query if use_gpu and torch.cuda.is_available(): self.device = torch.device("cuda") else: self.device = torch.device("cpu") self.embed_title = embed_title # Init & Load Encoders self.query_tokenizer = Tokenizer.load( pretrained_model_name_or_path=query_embedding_model, do_lower_case=True, use_fast=use_fast_tokenizers) self.query_encoder = LanguageModel.load( pretrained_model_name_or_path=query_embedding_model, language_model_class="DPRQuestionEncoder") self.passage_tokenizer = Tokenizer.load( pretrained_model_name_or_path=passage_embedding_model, do_lower_case=True, use_fast=use_fast_tokenizers) self.passage_encoder = LanguageModel.load( pretrained_model_name_or_path=passage_embedding_model, language_model_class="DPRContextEncoder") self.processor = TextSimilarityProcessor( tokenizer=self.query_tokenizer, passage_tokenizer=self.passage_tokenizer, max_seq_len_passage=self.max_seq_len_passage, max_seq_len_query=self.max_seq_len_query, label_list=["hard_negative", "positive"], metric="text_similarity_metric", embed_title=self.embed_title, num_hard_negatives=0, num_negatives=0) prediction_head = TextSimilarityHead( similarity_function=similarity_function) self.model = BiAdaptiveModel( language_model1=self.query_encoder, language_model2=self.passage_encoder, prediction_heads=[prediction_head], embeds_dropout_prob=0.1, lm1_output_types=["per_sequence"], lm2_output_types=["per_sequence"], device=self.device, ) self.model.connect_heads_with_processor(self.processor.tasks, require_labels=False)