def setUp(self): self.default_template = '[T] [T] {arbitrary} [T] {fields} [P]' self.default_config = AutoConfig.from_pretrained('bert-base-cased') self.default_tokenizer = AutoTokenizer.from_pretrained('bert-base-cased') utils.add_task_specific_tokens(self.default_tokenizer) self.default_instance = { 'arbitrary': 'does this', 'fields': 'work', 'label': 'and' }
def load_pretrained(model_name): """ Loads pretrained HuggingFace config/model/tokenizer, as well as performs required initialization steps to facilitate working with triggers. """ config = AutoConfig.from_pretrained(model_name) model = AutoModelWithLMHead.from_pretrained(model_name) model.eval() tokenizer = AutoTokenizer.from_pretrained(model_name, add_prefix_space=True) utils.add_task_specific_tokens(tokenizer) return config, model, tokenizer
def test_collator(self): template = '[T] [T] {arbitrary} [T] {fields} [P]' tokenizer = AutoTokenizer.from_pretrained('bert-base-cased') config = AutoConfig.from_pretrained('bert-base-cased') utils.add_task_specific_tokens(tokenizer) templatizer = utils.TriggerTemplatizer( template, config, tokenizer, add_special_tokens=False ) collator = utils.Collator(pad_token_id=tokenizer.pad_token_id) instances = [ {'arbitrary': 'a', 'fields': 'the', 'label': 'hot'}, {'arbitrary': 'a a', 'fields': 'the the', 'label': 'cold'} ] templatized_instances = [templatizer(x) for x in instances] loader = DataLoader( templatized_instances, batch_size=2, shuffle=False, collate_fn=collator ) model_inputs, labels = next(iter(loader)) # Check results match our expectations expected_labels = torch.tensor([ tokenizer.encode('hot', add_special_tokens=False, add_prefix_space=True), tokenizer.encode('cold', add_special_tokens=False, add_prefix_space=True), ]) assert torch.equal(expected_labels, labels) expected_trigger_mask = torch.tensor([ [True, True, False, True, False, False, False, False], [True, True, False, False, True, False, False, False], ]) assert torch.equal(expected_trigger_mask, model_inputs['trigger_mask']) expected_predict_mask = torch.tensor([ [False, False, False, False, False, True, False, False], [False, False, False, False, False, False, False, True], ]) assert torch.equal(expected_predict_mask, model_inputs['predict_mask'])
def test_roberta(self): config = AutoConfig.from_pretrained('roberta-base') tokenizer = AutoTokenizer.from_pretrained('roberta-base') utils.add_task_specific_tokens(tokenizer) templatizer = utils.TriggerTemplatizer( self.default_template, config, tokenizer, add_special_tokens=False ) model_inputs, label = templatizer(self.default_instance) # Label should be mapped to its token id expected_label = torch.tensor([tokenizer.convert_tokens_to_ids([self.default_instance['label']])]) assert torch.equal(expected_label, label) # For BERT ouput is expected to have the following keys print(model_inputs) assert 'input_ids' in model_inputs assert 'attention_mask' in model_inputs # Test that the custom masks match our expectations expected_trigger_mask = torch.tensor( [[True, True, False, False, True, False, False]] ) assert torch.equal(expected_trigger_mask, model_inputs['trigger_mask']) expected_predict_mask = torch.tensor( [[False, False, False, False, False, False, True]] ) assert torch.equal(expected_predict_mask, model_inputs['predict_mask']) # Lastly, ensure [P] is replaced by a [MASK] token input_ids = model_inputs['input_ids'] predict_mask = model_inputs['predict_mask'] predict_token_id = input_ids[predict_mask].squeeze().item() assert predict_token_id == tokenizer.mask_token_id