def occlusion_sensitivity(self, value, index):
for classe in self.labels:
occlusion_sensitivity = OcclusionSensitivity()
grid = occlusion_sensitivity.explain((value, self.setY[index]),
self.model,
class_index=classe,
patch_size=20)
name = "n_{}_".format(index) + str(
classe) + "_occlusion_sensitivity.png"
occlusion_sensitivity.save(grid, ".", sg.PATH_OCCLUSION + name)
def test_should_produce_heatmap(convolutional_model, random_data, mocker):
mocker.patch(
"tf_explain.core.occlusion_sensitivity.grid_display",
return_value=mocker.sentinel.grid,
)
explainer = OcclusionSensitivity()
grid = explainer.explain(random_data, convolutional_model, 0, 10)
assert grid == mocker.sentinel.grid
def on_epoch_end(self, epoch, logs=None):
"""
Draw Occlusion Sensitivity outputs at each epoch end to Tensorboard.
Args:
epoch (int): Epoch index
logs (dict): Additional information on epoch
"""
explainer = OcclusionSensitivity()
grid = explainer.explain(
self.validation_data, self.model, self.class_index, self.patch_size
)
# Using the file writer, log the reshaped image.
with self.file_writer.as_default():
tf.summary.image("Occlusion Sensitivity", np.array([grid]), step=epoch)
def test_should_get_sensitivity_map(convolutional_model, random_data, mocker):
x, y = random_data
patch_size = 4
predict_return_value = np.ones((
math.ceil(x[0].shape[0] / patch_size) *
math.ceil(x[0].shape[1] / patch_size),
1,
)) * np.expand_dims([0.6, 0.4], axis=0)
convolutional_model.predict = mocker.MagicMock(
return_value=predict_return_value)
mocker.patch(
"tf_explain.core.occlusion_sensitivity.apply_grey_patch",
return_value=np.random.randint(
low=0, high=255, size=convolutional_model.inputs[0].shape[1:]),
)
mocker.patch("tf_explain.core.occlusion_sensitivity.cv2.resize",
side_effect=lambda x, _: x)
output = OcclusionSensitivity().get_sensitivity_map(
model=convolutional_model,
image=x[0],
class_index=0,
patch_size=patch_size)
expected_output = 0.4 * np.ones(
(x[0].shape[0] // patch_size, x[0].shape[1] // patch_size))
np.testing.assert_almost_equal(output, expected_output)
def visualise_occlusion_sensitivity(model,
class_label,
image_path,
datadir,
patch_size=3,
img_size=32):
# '''
# function: to visualize output after a specific layer of model
# param: layer_name - name of layer of model whose output is needed
# model - saved model
# returns: string file-path to outputplot
# '''
image = load_img(image_path, target_size=(img_size, img_size))
image = img_to_array(image)
explainer = OcclusionSensitivity()
grid = explainer.explain(([image], None), model, class_label, patch_size)
grid = cv.resize(grid, dsize=(10, 10))
plot_save_path = datadir + "OS.png"
cv.imwrite(plot_save_path, grid)
return plot_save_path
def save_all_explainer(validation_data,
model,
name_conv,
n_conv=1,
dir_save_im='./',
save_name='outputs'):
explainerGradCam = GradCAM()
explainerActiv = ExtractActivations()
explainerOccl = OcclusionSensitivity()
explainerSmoothGrad = SmoothGrad()
for i in range(1, n_conv + 1):
output = explainerActiv.explain(validation_data, model,
'{}_{}'.format(name_conv, i))
explainerActiv.save(output, dir_save_im,
'{}-activ-conv{}.jpg'.format(save_name, i))
output = explainerGradCam.explain(validation_data, model,
'{}_{}'.format(name_conv, i), 0)
explainerGradCam.save(output, dir_save_im,
'{}-gradCam0-conv{}.jpg'.format(save_name, i))
output = explainerSmoothGrad.explain(validation_data, model, 0)
explainerSmoothGrad.save(output, dir_save_im,
'{}-smooth0.jpg'.format(save_name))
output = explainerSmoothGrad.explain(validation_data, model, 1)
explainerSmoothGrad.save(output, dir_save_im,
'{}-smooth1.jpg'.format(save_name))
output = explainerOccl.explain(validation_data, model, 0, 5)
explainerOccl.save(output, dir_save_im,
'{}-occlSens0.jpg'.format(save_name))
output = explainerOccl.explain(validation_data, model, 1, 5)
explainerOccl.save(output, dir_save_im,
'{}-occlSens1.jpg'.format(save_name))
import tensorflow as tf
from tf_explain.core.occlusion_sensitivity import OcclusionSensitivity
IMAGE_PATH = './cat.jpg'
if __name__ == '__main__':
model = tf.keras.applications.resnet50.ResNet50(weights='imagenet',
include_top=True)
img = tf.keras.preprocessing.image.load_img(IMAGE_PATH,
target_size=(224, 224))
img = tf.keras.preprocessing.image.img_to_array(img)
model.summary()
data = ([img], None)
tabby_cat_class_index = 281
explainer = OcclusionSensitivity()
# Compute Occlusion Sensitivity for patch_size 20
grid = explainer.explain(data, model, tabby_cat_class_index, 20)
explainer.save(grid, '.', 'occlusion_sensitivity_20.png')
# Compute Occlusion Sensitivity for patch_size 10
grid = explainer.explain(data, model, tabby_cat_class_index, 10)
explainer.save(grid, '.', 'occlusion_sensitivity_10.png')
'Brown headed cowbird F', 'brown headed cowbird M', 'Carolina wren',
'Common Grakle', 'Downy woodpecker', 'Eatern Bluebird',
'Eu starling on-duty Ad', 'Eu starling off-duty Ad', 'House finch M',
'House finch F', 'House sparrow F/Im', 'House sparrow M', 'Mourning dove',
'Caridal M', 'Cardinal F', 'Norhtern flicker (red)', 'Pileated woodpecker',
'Red winged blackbird F/Im', 'Red winged blackbird M', 'Squirrel!',
'Tufted titmouse', 'White breasted nuthatch'
]
file_list = []
label_list = []
filename_classes = "C:/Users/alert/Google Drive/ML/Databases/Birds_dB/Mappings/minimal_bird_list.txt"
LIST_OF_CLASSES = [line.strip() for line in open(filename_classes, 'r')]
explainer = GradCAM()
explainer_occ = OcclusionSensitivity()
model = load_model("E:\\KerasOutput\\run_2019_11_25_07_33\\my_keras_model.h5")
model.summary()
testDir = "C:\\Users\\alert\\Google Drive\\ML\\Electric Bird Caster\\Classified\Carolina wren\\"
files = os.listdir(testDir)
shuffle(files)
# f = "2019-11-12_07-25-14_840.jpg"
for f in files:
img_pil = image.load_img(path=testDir + f, target_size=(224, 224, 3))
img = image.img_to_array(img_pil)
# print( '\n --- Warning - estimate for LRP might not be correct! --- \n --- Reason: estimated output at pixel={}. ---'.format(pixel_value))
my_data = ( [my_sample], None )
# CV colormaps to choose from:
# See https://docs.opencv.org/master/d3/d50/group__imgproc__colormap.html#gga9a805d8262bcbe273f16be9ea2055a65afdb81862da35ea4912a75f0e8f274aeb
# for list of CV colormaps
# COLORMAP_HOT # white to red
# COLORMAP_BONE # white to gray
# COLORMAP_PINK # white to reddish
if method_number==1: # Occlusion method - results were disappointing.
########### Method: Occlusion ################
explainer = OcclusionSensitivity() # define which type of heatmap we want
occlusion_patch_width = 20
heatmap = explainer.explain( my_data, model, my_index, occlusion_patch_width, colormap=cv2.COLORMAP_PINK )
my_title_text = 'Occlusion - patch = ' + repr(occlusion_patch_width) + ' pixels'
my_file_text = 'tf_explain_occlusion' + '_' + repr(occlusion_patch_width) #+ repr{my_row} + '{}_P{}_{}'.format('%i %i %i')
elif method_number==2:
########### Method: Grad CAM ################
### Apply GradCAM (class activation map) to the last convolution layer of the network.
explainer = GradCAM()
# find last convolution layer in model (typically 1x1 convolution)
n_layers = len(model.layers)
layer_names = [layer.name for layer in model.layers]
my_layer_name = '' # start with empty string
def occlSensitivity(X, model, class_index, patch_size):
explainerOccl = OcclusionSensitivity()
outputs = explainerOccl.explain((X, None), model, class_index, patch_size)
return cv2.cvtColor(outputs, cv2.COLOR_RGB2BGR)
#################################### Load Data #####################################
folder = './drive/MyDrive/processed_data/'
te_data = np.load(folder+'data_test.npy')
FOV = np.load(folder+'FOV_te.npy')
te_mask = np.load(folder+'mask_test.npy')
te_data = np.expand_dims(te_data, axis=3)
print('Dataset loaded')
#te_data2 = dataset_normalized(te_data)
te_data2 = te_data /255.
explainer = GradCAM()
dataaa = (te_data2[1].reshape(1,512,512,1),None)
grid = explainer.explain(dataaa, model, class_index=1)
explainer.save(grid, ".", "grad_cam.png")
explainer = OcclusionSensitivity()
grid = explainer.explain(dataaa, model, class_index=1)
explainer.save(grid, ".", "occ_cam.png")
def explain_tfexplain():
global graph
data = {"success": "failed"}
# # ensure an image was properly uploaded to our endpoint
if flask.request.method == "POST":
if flask.request.form.get("image"):
explainer = request.args.get("explainer")
#with graph.as_default():
model_path = flask.request.form.get("model_path")
model = load_model(model_path)
# # read the image in PIL format
image64 = flask.request.form.get("image")
image = base64.b64decode(image64)
image = Image.open(io.BytesIO(image))
image = prepare_image(image, target=(224, 224))
image = image*(1./255)
#img = tf.keras.preprocessing.image.img_to_array(image)
prediction = model.predict(image)
topClass = getTopXpredictions(prediction, 1)
print(topClass[0])
image = np.squeeze(image)
if explainer == "GRADCAM":
im = ([image], None)
from tf_explain.core.grad_cam import GradCAM
exp = GradCAM()
imgFinal = exp.explain(im, model, class_index=topClass[0][0])
#exp.save(imgFinal, ".", "grad_cam.png")
elif explainer == "OCCLUSIONSENSITIVITY":
im = ([image], None)
from tf_explain.core.occlusion_sensitivity import OcclusionSensitivity
exp = OcclusionSensitivity()
imgFinal = exp.explain(im, model,class_index=topClass[0][0], patch_size=10)
#exp.save(imgFinal, ".", "grad_cam.png")
elif explainer == "GRADIENTSINPUTS":
im = (np.array([image]), None)
from tf_explain.core.gradients_inputs import GradientsInputs
exp = GradientsInputs()
imgFinal = exp.explain(im, model, class_index=topClass[0][0])
#exp.save(imgFinal, ".", "gradients_inputs.png")
elif explainer == "VANILLAGRADIENTS":
im = (np.array([image]), None)
from tf_explain.core.vanilla_gradients import VanillaGradients
exp = VanillaGradients()
imgFinal = exp.explain(im, model, class_index=topClass[0][0])
#exp.save(imgFinal, ".", "gradients_inputs.png")
elif explainer == "SMOOTHGRAD":
im = (np.array([image]), None)
from tf_explain.core.smoothgrad import SmoothGrad
exp = SmoothGrad()
imgFinal = exp.explain(im, model, class_index=topClass[0][0])
#exp.save(imgFinal, ".", "gradients_inputs.png")
elif explainer == "INTEGRATEDGRADIENTS":
im = (np.array([image]), None)
from tf_explain.core.integrated_gradients import IntegratedGradients
exp = IntegratedGradients()
imgFinal = exp.explain(im, model, class_index=topClass[0][0])
#exp.save(imgFinal, ".", "gradients_inputs.png")
elif explainer == "ACTIVATIONVISUALIZATION":
#need some solution to find out and submit layers name
im = (np.array([image]), None)
from tf_explain.core.activations import ExtractActivations
exp = ExtractActivations()
imgFinal = exp.explain(im, model, layers_name=["activation_1"])
#exp.save(imgFinal, ".", "gradients_inputs.png")
img = pilimage.fromarray(imgFinal)
imgByteArr = inputoutput.BytesIO()
img.save(imgByteArr, format='JPEG')
imgByteArr = imgByteArr.getvalue()
img64 = base64.b64encode(imgByteArr)
img64_string = img64.decode("utf-8")
data["explanation"] = img64_string
data["prediction"] = str(topClass[0][0])
data["prediction_score"] = str(topClass[0][1])
data["success"] = "success"
return flask.Response(json.dumps(data), mimetype="text/plain")
def write():
st.title(' Face Mask Detector')
net = load_face_detector_and_model()
model = load_cnn_model()
selected_option = st.radio("Choose", ('File', 'Webcam'))
if selected_option == 'File':
#uploaded_image = st.sidebar.file_uploader("Choose a JPG file", type="jpg")
uploaded_image = st.sidebar.file_uploader("Choose a JPG file",
type=FILE_TYPES)
confidence_value = st.sidebar.slider('Confidence:', 0.0, 1.0, 0.5, 0.1)
if uploaded_image:
image1 = Image.open(uploaded_image)
st.sidebar.image(image1,
caption='Uploaded Image.',
use_column_width=True)
#st.sidebar.info('Uploaded image:')
#st.sidebar.image(uploaded_image, width=240)
#f = open(uploaded_image, 'rb')
#file = st.file_uploader("Upload file", type=FILE_TYPES)
show_file = st.empty()
if not uploaded_image:
show_file.info("Please upload a file of type: " +
", ".join(FILE_TYPES))
return
file_type = get_file_type(uploaded_image)
if file_type == FileType.IMAGE:
show_file.image(image1)
elif file_type == FileType.PYTHON:
st.code(uploaded_image.getvalue())
else:
data = pd.read_csv(uploaded_image)
st.dataframe(data.head(10))
f = open(get_file_name(uploaded_image), 'rb')
img_bytes = f.read()
f.close()
grad_cam_button = st.sidebar.button('Grad CAM')
patch_size_value = st.sidebar.slider('Patch size:', 10, 90, 20, 10)
occlusion_sensitivity_button = st.sidebar.button(
'Occlusion Sensitivity')
image = cv2.imdecode(np.fromstring(img_bytes, np.uint8), 1)
#image = cv2.imdecode(np.fromstring(uploaded_image.read(), np.uint8), 1)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
orig = image.copy()
(h, w) = image.shape[:2]
blob = cv2.dnn.blobFromImage(image, 1.0, (300, 300),
(104.0, 177.0, 123.0))
net.setInput(blob)
detections = net.forward()
for i in range(0, detections.shape[2]):
confidence = detections[0, 0, i, 2]
if confidence > confidence_value:
box = detections[0, 0, i, 3:7] * np.array([w, h, w, h])
(startX, startY, endX, endY) = box.astype("int")
(startX, startY) = (max(0, startX), max(0, startY))
(endX, endY) = (min(w - 1, endX), min(h - 1, endY))
face = image[startY:endY, startX:endX]
face = cv2.cvtColor(face, cv2.COLOR_BGR2RGB)
face = cv2.resize(face, (224, 224))
face = img_to_array(face)
face = preprocess_input(face)
expanded_face = np.expand_dims(face, axis=0)
(mask, withoutMask) = model.predict(expanded_face)[0]
predicted_class = 0
label = "No Mask"
if mask > withoutMask:
label = "Mask"
predicted_class = 1
color = (0, 255, 0) if label == "Mask" else (0, 0, 255)
label = "{}: {:.2f}%".format(label,
max(mask, withoutMask) * 100)
cv2.putText(image, label, (startX, startY - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.45, color, 2)
cv2.rectangle(image, (startX, startY), (endX, endY), color,
2)
st.image(image, width=640)
st.write('### ' + label)
if grad_cam_button:
data = ([face], None)
explainer = GradCAM()
grad_cam_grid = explainer.explain(data,
model,
class_index=predicted_class,
layer_name="Conv_1")
st.image(grad_cam_grid)
if occlusion_sensitivity_button:
data = ([face], None)
explainer = OcclusionSensitivity()
sensitivity_occlusion_grid = explainer.explain(
data, model, predicted_class, patch_size_value)
st.image(sensitivity_occlusion_grid)
# PROGRAM FOR WEB CAM
if selected_option == 'Webcam':
labels_dict = {0: 'without_mask', 1: 'with_mask'}
color_dict = {0: (0, 0, 255), 1: (0, 255, 0)}
size = 4
webcam = cv2.VideoCapture(0) #Use camera 0
st.write("Webcam On")
stframe_cam = st.empty()
# We load the xml file
classifier = cv2.CascadeClassifier(
'src/pages/Services/frecog/haarcascade_frontalface_default.xml')
while True:
(rval, im) = webcam.read()
stframe_cam.image(im)
# st.write("Webcam Read")
#if im:
#ret, framecar = vf.read()
im = cv2.flip(im, 1, 1) #Flip to act as a mirror
# Resize the image to speed up detection
mini = cv2.resize(im, (im.shape[1] // size, im.shape[0] // size))
# detect MultiScale / faces
faces = classifier.detectMultiScale(mini)
# Draw rectangles around each face
for f in faces:
(x, y, w, h) = [v * size
for v in f] #Scale the shapesize backup
#Save just the rectangle faces in SubRecFaces
face_img = im[y:y + h, x:x + w]
resized = cv2.resize(face_img, (150, 150))
normalized = resized / 255.0
reshaped = np.reshape(normalized, (1, 150, 150, 3))
reshaped = np.vstack([reshaped])
result = model.predict(reshaped)
#print(result)
label = np.argmax(result, axis=1)[0]
cv2.rectangle(im, (x, y), (x + w, y + h), color_dict[label], 2)
cv2.rectangle(im, (x, y - 40), (x + w, y), color_dict[label],
-1)
cv2.putText(im, labels_dict[label], (x, y - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.8, (255, 255, 255), 2)
# Show the image
stframe_cam.image('LIVE', im)
#cv2.imshow('LIVE', im)
key = cv2.waitKey(10)
# if Esc key is press then break out of the loop
if key == 27: #The Esc key
break
# Stop video
webcam.release()
# Close all started windows
cv2.destroyAllWindows()
#write()
#uploaded_image.close()