def _predict(input_face, device):
"""
Facial expression recognition. Classifies the pre-processed input image with ESR-9.
:param input_face: (ndarray) input image.
:param device: runs the classification on CPU or GPU
:return: Lists of emotions and affect values including the ensemble predictions based on plurality.
"""
global _ESR_9
if _ESR_9 is None:
_ESR_9 = ESR(device)
to_return_emotion = []
to_return_emotion_idx = []
to_return_affect = None
# Recognizes facial expression
emotion, affect = _ESR_9(input_face)
# Computes ensemble prediction for affect
# Converts from Tensor to ndarray
affect = np.array([a[0].cpu().detach().numpy() for a in affect])
# Normalizes arousal
affect[:, 1] = np.clip((affect[:, 1] + 1) / 2.0, 0, 1)
# Computes mean arousal and valence as the ensemble prediction
ensemble_affect = np.expand_dims(np.mean(affect, 0), axis=0)
# Concatenates the ensemble prediction to the list of affect predictions
to_return_affect = np.concatenate((affect, ensemble_affect), axis=0)
# Computes ensemble prediction concerning emotion labels
# Converts from Tensor to ndarray
emotion = np.array([e[0].cpu().detach().numpy() for e in emotion])
# Gets number of classes
num_classes = emotion.shape[1]
# Computes votes and add label to the list of emotions
emotion_votes = np.zeros(num_classes)
for e in emotion:
e_idx = np.argmax(e)
to_return_emotion_idx.append(e_idx)
to_return_emotion.append(udata.AffectNetCategorical.get_class(e_idx))
emotion_votes[e_idx] += 1
# Concatenates the ensemble prediction to the list of emotion predictions
to_return_emotion.append(
udata.AffectNetCategorical.get_class(np.argmax(emotion_votes)))
return to_return_emotion, to_return_affect, to_return_emotion_idx
def load(device_to_load, ensemble_size):
# Load ESR-9
esr_9 = ESR(device_to_load)
loaded_model = Ensemble()
loaded_model.branches = []
# Load the base of the network
loaded_model.base = esr_9.base
# Load branches
for i in range(ensemble_size):
loaded_model_branch = Branch()
loaded_model_branch.conv1 = esr_9.convolutional_branches[i].conv1
loaded_model_branch.conv2 = esr_9.convolutional_branches[i].conv2
loaded_model_branch.conv3 = esr_9.convolutional_branches[i].conv3
loaded_model_branch.conv4 = esr_9.convolutional_branches[i].conv4
loaded_model_branch.bn1 = esr_9.convolutional_branches[i].bn1
loaded_model_branch.bn2 = esr_9.convolutional_branches[i].bn2
loaded_model_branch.bn3 = esr_9.convolutional_branches[i].bn3
loaded_model_branch.bn4 = esr_9.convolutional_branches[i].bn4
loaded_model_branch.fc = esr_9.convolutional_branches[i].fc
loaded_model.branches.append(loaded_model_branch)
return loaded_model
def __init__(self, device="cuda:0"):
super(EmotionRecognition, self).__init__()
self.device = device
self.model = _ESR_9 = ESR(device)
def load(device_to_load, ensemble_size):
# Load ESR-9
esr_9 = ESR(device_to_load)
loaded_model = Ensemble()
loaded_model.branches = []
# Load the base of the network
loaded_model.base = esr_9.base
# Base no trainable
for p in loaded_model.base.conv1.parameters():
p.requires_grad = False
for p in loaded_model.base.conv2.parameters():
p.requires_grad = False
for p in loaded_model.base.conv3.parameters():
p.requires_grad = False
for p in loaded_model.base.conv4.parameters():
p.requires_grad = False
for p in loaded_model.base.bn1.parameters():
p.requires_grad = False
for p in loaded_model.base.bn2.parameters():
p.requires_grad = False
for p in loaded_model.base.bn3.parameters():
p.requires_grad = False
for p in loaded_model.base.bn4.parameters():
p.requires_grad = False
# Load branches
for i in range(ensemble_size):
loaded_model_branch = Branch()
loaded_model_branch.conv1 = esr_9.convolutional_branches[i].conv1
loaded_model_branch.conv2 = esr_9.convolutional_branches[i].conv2
loaded_model_branch.conv3 = esr_9.convolutional_branches[i].conv3
loaded_model_branch.conv4 = esr_9.convolutional_branches[i].conv4
loaded_model_branch.bn1 = esr_9.convolutional_branches[i].bn1
loaded_model_branch.bn2 = esr_9.convolutional_branches[i].bn2
loaded_model_branch.bn3 = esr_9.convolutional_branches[i].bn3
loaded_model_branch.bn4 = esr_9.convolutional_branches[i].bn4
loaded_model_branch.fc = esr_9.convolutional_branches[i].fc
# Branch no trainable, but last layer
for p in loaded_model_branch.conv1.parameters():
p.requires_grad = False
for p in loaded_model_branch.conv2.parameters():
p.requires_grad = False
for p in loaded_model_branch.conv3.parameters():
p.requires_grad = False
for p in loaded_model_branch.conv4.parameters():
p.requires_grad = False
for p in loaded_model_branch.bn1.parameters():
p.requires_grad = False
for p in loaded_model_branch.bn2.parameters():
p.requires_grad = False
for p in loaded_model_branch.bn3.parameters():
p.requires_grad = False
for p in loaded_model_branch.bn4.parameters():
p.requires_grad = False
for p in loaded_model_branch.fc.parameters():
p.requires_grad = False
loaded_model.branches.append(loaded_model_branch)
return loaded_model