def _examples():
""" Example for how to use this explainer
"""
# read data to try
data_path = r"hm-data/"
labels = utils.read_labels(data_path + "train.jsonl", True)
ids = [28061]
target_labels = [l for l in labels if l['id'] in ids]
print(f" target_labels = {target_labels}")
target_images, target_texts = utils.parse_labels(target_labels,
img_to_array=True,
separate_outputs=True)
# model to explain
model = MMBT.from_pretrained("mmbt.hateful_memes.images")
# Explainer hyper params
max_evals = 100
batch_size = 50
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model.to(device)
# test default partition algo
explainer = Explainer(model, max_evals=max_evals, batch_size=batch_size)
# text_shap_values = explainer.explain(target_images, target_texts, "text_only")
image_shap_values = explainer.explain(target_images, target_texts,
"image_only")
# plots
# explainer.text_plot(text_shap_values)
explainer.image_plot(image_shap_values)
def setup_model(user_model, model_type, model_path):
if user_model == "no_model":
try:
if model_type == "MMBT":
model = MMBT.from_pretrained("mmbt.hateful_memes.images")
elif model_type == "LateFusion":
model = LateFusion.from_pretrained("late_fusion.hateful_memes")
elif model_type == "ViLBERT":
model = ViLBERT.from_pretrained(
"vilbert.finetuned.hateful_memes.from_cc_original"
)
else: # visual bert
model = VisualBERT.from_pretrained(
"visual_bert.finetuned.hateful_memes.from_coco"
)
except:
raise InputError(
"Sorry, having trouble opening the models we provided, please try again later."
)
elif user_model == "mmf":
try:
if model_type == "MMBT":
model = MMBT.from_pretrained(model_path)
elif model_type == "LateFusion":
model = LateFusion.from_pretrained(model_path)
elif model_type == "ViLBERT":
model = ViLBERT.from_pretrained(model_path)
else:
model = VisualBERT.from_pretrained(model_path)
except:
raise InputError(
"Sorry, we cannot open the mmf checkpoint you uploaded. It should be an .ckpt file saved from the mmf trainer."
)
elif user_model == "onnx":
model = ONNXInterface(model_path, model_type)
else:
raise InputError("Please select a model upload type")
return model
def test_mmbt_hm_interface(self):
model = MMBT.from_pretrained("mmbt.hateful_memes.images")
result = model.classify("https://i.imgur.com/tEcsk5q.jpg",
"look how many people love you")
self.assertEqual(result["label"], 0)
np.testing.assert_almost_equal(result["confidence"], 0.9993, decimal=4)
result = model.classify("https://i.imgur.com/tEcsk5q.jpg",
"they have the privilege")
self.assertEqual(result["label"], 0)
np.testing.assert_almost_equal(result["confidence"], 0.9777, decimal=4)
result = model.classify("https://i.imgur.com/tEcsk5q.jpg",
"hitler and jews")
self.assertEqual(result["label"], 1)
np.testing.assert_almost_equal(result["confidence"], 0.6342, decimal=4)
def setup_model(user_model, model_type, model_path):
if user_model == "no_model":
if model_type == "mmbt":
model = MMBT.from_pretrained("mmbt.hateful_memes.images")
elif model_type == "fusion":
fusion = LateFusion.from_pretrained("late_fusion.hateful_memes")
elif model_type == "vilbert":
vilbert = ViLBERT.from_pretrained("vilbert.finetuned.hateful_memes.direct")
elif model_type == "visual_bert"
visual_bert_model = VisualBERT.from_pretrained("visual_bert.finetuned.hateful_memes.direct")
elif user_model == "mmf":
if model_type == "mmbt":
model = MMBT.from_pretrained(model_path)
elif model_type == "fusion":
fusion = LateFusion.from_pretrained(model_path)
elif model_type == "vilbert":
vilbert = ViLBERT.from_pretrained(model_path)
elif model_type == "visual_bert"
visual_bert_model = VisualBERT.from_pretrained(model_path)
# elif user_model == "onnx": ?????
return model
def main():
image_path = input("enter your image path : ")
text = input("enter your text : ")
model = MMBT.from_pretrained("mmbt.hateful_memes.images")
model.to(torch.device("cuda:0" if torch.cuda.is_available() else "cpu"))
image_tensor = image2tensor(image_path)
mask_, hist_, output_tensor, txt_summary, text_explaination = multi_extremal_perturbation(
model,
image_tensor,
image_path,
text,
0, # 0 non hateful 1 hateful
max_iter=50,
areas=[0.12],
)
return output_tensor, txt_summary, text_explaination
def classify(self,image,text_input, image_tensor = None):
'''
Args:
image_path: directory of input image
text_input : the text input Str
image_tensor : the image torch.tensor with size (1,3,224,224)
Returns :
label of model prediction and the corresponding confidence
'''
scoreFlag = False
if image_tensor != None:
scoreFlag = True
logits = self.onnx_model_forward(image_tensor,text_input)
else:
p = transforms.Compose([transforms.Scale((224,224))])
image,i = imsc(p(image),quiet=True)
image_tensor = torch.reshape(image, (1,3,224,224))
logits = self.onnx_model_forward(image_tensor,text_input)
if list(torch.tensor(logits).size()) != [1, 2]:
if self.defaultmodel == None:
self.defaultmodel = MMBT.from_pretrained("mmbt.hateful_memes.images")
self.defaultmodel.to(self.device)
logits = self.defaultmodel.classify(image, text_input, image_tensor=torch.squeeze(image_tensor.to(self.device), 0))
scores = nn.functional.softmax(torch.tensor(logits), dim=1)
if scoreFlag == True:
return scores
confidence, label = torch.max(scores, dim=1)
return {"label": label.item(), "confidence": confidence.item()}
def global_data():
max_evals = 2
batch_size = 1
model = MMBT.from_pretrained("mmbt.hateful_memes.images")
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model.to(device)
labels = utils.read_labels(DATA_PATH, True)
# ids = [5643] # single input
ids = [5643, 6937] # multiple inputs - tested
target_labels = [l for l in labels if l["id"] in ids]
target_images, target_texts = utils.parse_labels(
target_labels, img_to_array=True, separate_outputs=True
)
outputs = model_outputs(model, target_images, target_texts)
return {
"max_evals": max_evals,
"batch_size": batch_size,
"model": model,
"target_images": target_images,
"target_texts": target_texts,
"outputs": outputs,
}
