def _examples(): """ Example for how to use this explainer """ # read data to try data_path = r"hm-data/" labels = utils.read_labels(data_path + "train.jsonl", True) ids = [28061] target_labels = [l for l in labels if l['id'] in ids] print(f" target_labels = {target_labels}") target_images, target_texts = utils.parse_labels(target_labels, img_to_array=True, separate_outputs=True) # model to explain model = MMBT.from_pretrained("mmbt.hateful_memes.images") # Explainer hyper params max_evals = 100 batch_size = 50 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") model.to(device) # test default partition algo explainer = Explainer(model, max_evals=max_evals, batch_size=batch_size) # text_shap_values = explainer.explain(target_images, target_texts, "text_only") image_shap_values = explainer.explain(target_images, target_texts, "image_only") # plots # explainer.text_plot(text_shap_values) explainer.image_plot(image_shap_values)
def setup_model(user_model, model_type, model_path): if user_model == "no_model": try: if model_type == "MMBT": model = MMBT.from_pretrained("mmbt.hateful_memes.images") elif model_type == "LateFusion": model = LateFusion.from_pretrained("late_fusion.hateful_memes") elif model_type == "ViLBERT": model = ViLBERT.from_pretrained( "vilbert.finetuned.hateful_memes.from_cc_original" ) else: # visual bert model = VisualBERT.from_pretrained( "visual_bert.finetuned.hateful_memes.from_coco" ) except: raise InputError( "Sorry, having trouble opening the models we provided, please try again later." ) elif user_model == "mmf": try: if model_type == "MMBT": model = MMBT.from_pretrained(model_path) elif model_type == "LateFusion": model = LateFusion.from_pretrained(model_path) elif model_type == "ViLBERT": model = ViLBERT.from_pretrained(model_path) else: model = VisualBERT.from_pretrained(model_path) except: raise InputError( "Sorry, we cannot open the mmf checkpoint you uploaded. It should be an .ckpt file saved from the mmf trainer." ) elif user_model == "onnx": model = ONNXInterface(model_path, model_type) else: raise InputError("Please select a model upload type") return model
def test_mmbt_hm_interface(self): model = MMBT.from_pretrained("mmbt.hateful_memes.images") result = model.classify("https://i.imgur.com/tEcsk5q.jpg", "look how many people love you") self.assertEqual(result["label"], 0) np.testing.assert_almost_equal(result["confidence"], 0.9993, decimal=4) result = model.classify("https://i.imgur.com/tEcsk5q.jpg", "they have the privilege") self.assertEqual(result["label"], 0) np.testing.assert_almost_equal(result["confidence"], 0.9777, decimal=4) result = model.classify("https://i.imgur.com/tEcsk5q.jpg", "hitler and jews") self.assertEqual(result["label"], 1) np.testing.assert_almost_equal(result["confidence"], 0.6342, decimal=4)
def setup_model(user_model, model_type, model_path): if user_model == "no_model": if model_type == "mmbt": model = MMBT.from_pretrained("mmbt.hateful_memes.images") elif model_type == "fusion": fusion = LateFusion.from_pretrained("late_fusion.hateful_memes") elif model_type == "vilbert": vilbert = ViLBERT.from_pretrained("vilbert.finetuned.hateful_memes.direct") elif model_type == "visual_bert" visual_bert_model = VisualBERT.from_pretrained("visual_bert.finetuned.hateful_memes.direct") elif user_model == "mmf": if model_type == "mmbt": model = MMBT.from_pretrained(model_path) elif model_type == "fusion": fusion = LateFusion.from_pretrained(model_path) elif model_type == "vilbert": vilbert = ViLBERT.from_pretrained(model_path) elif model_type == "visual_bert" visual_bert_model = VisualBERT.from_pretrained(model_path) # elif user_model == "onnx": ????? return model
def main(): image_path = input("enter your image path : ") text = input("enter your text : ") model = MMBT.from_pretrained("mmbt.hateful_memes.images") model.to(torch.device("cuda:0" if torch.cuda.is_available() else "cpu")) image_tensor = image2tensor(image_path) mask_, hist_, output_tensor, txt_summary, text_explaination = multi_extremal_perturbation( model, image_tensor, image_path, text, 0, # 0 non hateful 1 hateful max_iter=50, areas=[0.12], ) return output_tensor, txt_summary, text_explaination
def classify(self,image,text_input, image_tensor = None): ''' Args: image_path: directory of input image text_input : the text input Str image_tensor : the image torch.tensor with size (1,3,224,224) Returns : label of model prediction and the corresponding confidence ''' scoreFlag = False if image_tensor != None: scoreFlag = True logits = self.onnx_model_forward(image_tensor,text_input) else: p = transforms.Compose([transforms.Scale((224,224))]) image,i = imsc(p(image),quiet=True) image_tensor = torch.reshape(image, (1,3,224,224)) logits = self.onnx_model_forward(image_tensor,text_input) if list(torch.tensor(logits).size()) != [1, 2]: if self.defaultmodel == None: self.defaultmodel = MMBT.from_pretrained("mmbt.hateful_memes.images") self.defaultmodel.to(self.device) logits = self.defaultmodel.classify(image, text_input, image_tensor=torch.squeeze(image_tensor.to(self.device), 0)) scores = nn.functional.softmax(torch.tensor(logits), dim=1) if scoreFlag == True: return scores confidence, label = torch.max(scores, dim=1) return {"label": label.item(), "confidence": confidence.item()}
def global_data(): max_evals = 2 batch_size = 1 model = MMBT.from_pretrained("mmbt.hateful_memes.images") device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") model.to(device) labels = utils.read_labels(DATA_PATH, True) # ids = [5643] # single input ids = [5643, 6937] # multiple inputs - tested target_labels = [l for l in labels if l["id"] in ids] target_images, target_texts = utils.parse_labels( target_labels, img_to_array=True, separate_outputs=True ) outputs = model_outputs(model, target_images, target_texts) return { "max_evals": max_evals, "batch_size": batch_size, "model": model, "target_images": target_images, "target_texts": target_texts, "outputs": outputs, }