def predict_class(self, samples, sample_poses=None):
"""
first use "is_guaranteed_new_class"!
:param samples:
:param sample_poses:
:return: inconsistent, class, confidence
"""
# disable sample weights
sample_weight = None
# individual classifier predictions (binary)
predictions = {}
true_pos_rate = 0.7
true_pos_thresh = math.ceil(true_pos_rate*len(samples))
false_pos_rate = 0.4
false_pos_thresh = math.floor(false_pos_rate * len(samples))
target_positive_classes = []
true_positives_rates = []
false_positives = []
pos_matching_confidence = []
neg_matching_confidence = []
# select classes in range
classes_in_range = self.data_controller.classes_in_range(samples=samples, metric='euclidean', thresh=1.25)
if not classes_in_range:
log.info('db', "No classes in range...")
return True, -1, 1
for class_id, cls in self.classifiers.iteritems():
# only consider near classes
if class_id in classes_in_range:
# binary prediction
predictions[class_id], matching_confidences = cls.predict(samples, samples_poses=sample_poses)
true_positive_samples = np.count_nonzero(predictions[class_id] == 1)
# count certain detections
if true_positive_samples >= true_pos_thresh:
target_positive_classes.append(class_id)
true_positives_rates.append(true_positive_samples)
pos_matching_confidence.append(matching_confidences)
elif true_positive_samples >= false_pos_thresh:
# not true class - check if too many false positives
false_positives.append(class_id)
neg_matching_confidence.append(matching_confidences)
# else:
# false_positive_samples = np.count_nonzero(predictions[class_id] == -1)
# # count uncertain detections
# if false_positive_samples >= false_pos_thresh:
# false_positives.append(class_id)
is_consistent = True
target_class = None
safe_weight = 7
print "... T_fp: {}, T_tp: {} | fp ids: {}, tp ids: {}".format(false_pos_thresh, true_pos_thresh, false_positives, target_positive_classes)
confidence = 1.0
decision_weights = np.repeat(99., len(samples))
# check for inconsistent predictions
if len(false_positives) == 0:
if len(target_positive_classes) == 0:
# new class!
target_class = -1
# TODO: confidence not implemented yet! Build additive score
elif len(target_positive_classes) == 1:
# single target class
target_class = target_positive_classes[0]
decision_weights = pos_matching_confidence[0]
else:
# multiple target classes
is_consistent = False # not a valid result
best_index = np.argmax(true_positives_rates)
target_class = target_positive_classes[best_index]
decision_weights = pos_matching_confidence[best_index]
else:
is_consistent = False # not a valid/safe result
if len(target_positive_classes) == 0:
# new class!
target_class = -1
# TODO: confidence not implemented yet! Build additive score
elif len(target_positive_classes) == 1:
# target class
target_class = target_positive_classes[0]
decision_weights = pos_matching_confidence[0]
else:
best_index = np.argmax(true_positives_rates)
target_class = target_positive_classes[best_index]
decision_weights = pos_matching_confidence[best_index]
# TODO: not active right now
# if is_consistent and False:
# # check for strong samples with inconsistent predictions
# mask = sample_weight > safe_weight
# if np.count_nonzero(mask):
# # check if safe samples predict wrong class
# for class_id, pred in predictions.iteritems():
# if class_id == target_class:
# # false negative
# if np.count_nonzero(pred[mask] < 0):
# is_consistent = False
# break
# else:
# # false positive
# if np.count_nonzero(pred[mask] > 0):
# is_consistent = False
# break
# calculate confidence
# print "decision_weights: ", decision_weights
# print "pos_matching_confidence: ", pos_matching_confidence
print "==== decision_weights: ", ["%0.1f" % i for i in decision_weights]
if target_class == -1:
# TODO: not implemented yet! Build additive score
confidence = 1.0
elif target_class > 0:
# combine confidence with binary decision (weighted average)
confidence = self.calc_normalized_positive_confidence(predictions[target_class], weights=decision_weights)
else:
confidence = 1.0
print "---- Prediction: ", predictions
print "---- Target class decision: {} / conf: {} / TP: {}, FP: {} / min. TP: {} max. FP: {}".format(target_class, confidence, len(target_positive_classes), len(false_positives), true_pos_thresh, false_pos_thresh)
# confidence: 1...100 (full conf)
return is_consistent, target_class, confidence
def print_embedding_status(self):
log.info('db', "Current embeddings:")
for user_id, embeddings in self.__class_samples.iteritems():
log.info('db', " User" + str(user_id) + ": " + str(len(embeddings)) + " representations")
def get_weighted_score(self, test_samples, test_poses, ref_samples,
ref_poses):
assert test_samples.ndim == 2
assert ref_samples.ndim == 2
dist_lookup = pairwise_distances(test_samples,
ref_samples,
metric='euclidean')
# print np.shape(dist_lookup[0])
factors = []
sample_weights = []
# if only one sample: cannot calculate abof
if len(ref_samples) < 3:
log.severe(
'Cannot calculate ABOF with {} reference samples (variance calculation needs at least 3 reference points)'
.format(len(ref_samples)))
raise Exception
for i_sample, A in enumerate(test_samples):
factor_list = []
weight_list = []
for i in range(len(ref_samples)):
# select first point in reference set
B = ref_samples[i]
# distance
AB = dist_lookup[i_sample][i]
for j in range(i + 1):
if j == i: # ensure B != C
continue
# select second point in reference set
C = ref_samples[j]
# distance
AC = dist_lookup[i_sample][j]
if np.array_equal(B, C):
sys.exit(
"Points are equal: B == C! Reference Set contains two times the same samples"
)
factor_list.append(1000)
print "Bi/Cj: {}/{}".format(i, j)
# sys.exit('ERROR\tangleBAC\tmath domain ERROR, |cos<AB, AC>| <= 1')
continue
angle_BAC = ABOD.angleBAC(A, B, C, AB, AC)
w1 = self.weight_gen.get_pose_weight(
test_poses[i_sample], ref_poses[i])
w2 = self.weight_gen.get_pose_weight(
test_poses[i_sample], ref_poses[j])
weight_list.append(2. / float(w1 + w2)) # 1/(a+b)/2
# compute each element of variance list
try:
# apply weighting
if self.variant == 1:
tmp = angle_BAC / float(
math.pow(AB * AC, 2) * (w1 * w2))
elif self.variant == 2:
tmp = angle_BAC / float(
math.pow(AB * AC, 2) * (w1 + w2))
else:
tmp = angle_BAC / float(math.pow(AB * AC, 2))
except ZeroDivisionError:
log.severe(
"ERROR\tABOF\tfloat division by zero! Trying to predict training point?'"
)
tmp = 500
# sys.exit('ERROR\tABOF\tfloat division by zero! Trying to predict training point?')
factor_list.append(tmp)
# calculate weighted variance
if self.variant == 3:
weighted_average = np.average(factor_list,
weights=np.array(weight_list))
var = np.average((factor_list - weighted_average)**2)
elif self.variant == 4:
var = WeightedABOD.biased_weighted_var(np.array(factor_list),
np.array(weight_list),
weighted_average=False)
elif self.variant == 5:
var = WeightedABOD.biased_weighted_var(np.array(factor_list),
np.array(weight_list))
else:
var = np.var(np.array(factor_list))
factors.append(var)
# weight_list = np.repeat(1, len(factors))
sample_weights.append(np.average(weight_list))
return np.array(factors), np.array(sample_weights)
def get_score(test_samples, reference_set):
assert test_samples.ndim == 2
assert reference_set.ndim == 2
dist_lookup = pairwise_distances(test_samples,
reference_set,
metric='euclidean')
# print np.shape(dist_lookup[0])
factors = []
# if only one sample: cannot calculate abof
if len(reference_set) < 3:
log.severe(
'Cannot calculate ABOF with {} reference samples (variance calculation needs at least 3 reference points)'
.format(len(reference_set)))
raise Exception
for i_sample, A in enumerate(test_samples):
factor_list = []
for i in range(len(reference_set)):
# select first point in reference set
B = reference_set[i]
# distance
AB = dist_lookup[i_sample][i]
for j in range(i + 1):
if j == i: # ensure B != C
continue
# select second point in reference set
C = reference_set[j]
# distance
AC = dist_lookup[i_sample][j]
if np.array_equal(B, C):
print "Bi/Cj: {}/{}".format(i, j)
log.error(
"Points are equal: B == C! Assuming classification of training point"
)
sys.exit(
"Points are equal: B == C! Reference Set contains two times the same samples"
)
factor_list.append(1000)
# sys.exit('ERROR\tangleBAC\tmath domain ERROR, |cos<AB, AC>| <= 1')
continue
# angle_BAC = ABOD.angleBAC(A, B, C, AB, AC)
# angle_BAC = ABOD.angleFast(A-B, A-C)
vector_AB = B - A
vector_AC = C - A
# compute each element of variance list
try:
cos_similarity = np.dot(vector_AB,
vector_AC) / (AB * AC)
# apply weighting
tmp = cos_similarity / float(math.pow(AB * AC, 2))
except ZeroDivisionError:
log.severe(
"ERROR\tABOF\tfloat division by zero! Trying to predict training point?'"
)
tmp = 500
# sys.exit('ERROR\tABOF\tfloat division by zero! Trying to predict training point?')
factor_list.append(tmp)
factors.append(np.var(factor_list))
return np.array(factors)
def update(self, samples):
# init - add all samples if no data yet
if len(self.__data) == 0:
self.__data = samples
return
# =======================================
# 1. Reduce data/sample data
if self.dim_reduction > 0:
basis, mean, var = ExtractMaxVarComponents(self.__data,
self.dim_reduction)
self.log_expl_var.append(var)
else:
basis, mean = ExtractSubspace(self.__data, 0.8)
cluster_reduced = ProjectOntoSubspace(self.__data, mean, basis)
samples_reduced = ProjectOntoSubspace(samples, mean, basis)
dims = np.shape(cluster_reduced)
# select minimum data to build convex hull
# min_nr_elems = dims[1] + 4
if self.__verbose:
print "Reducing dimension: {}->{}".format(
np.shape(self.__data)[1], dims[1])
# =======================================
# 2. Calculate Convex Hull in subspace
data_hull = cluster_reduced # take all samples of data
hull = Delaunay(data_hull)
if self.__verbose:
print "Calculating data hull using {}/{} points".format(
len(data_hull), len(self.__data))
# =======================================
# 3. Select new samples from outside convex hull
if not self.__inverted:
inclusion_mask = np.array([
False if hull.find_simplex(sample) >= 0 else True
for sample in samples_reduced
])
if self.__verbose:
# Todo: use outside mask counting
nr_elems_outside_hull = np.sum([
0 if hull.find_simplex(sample) >= 0 else 1
for sample in samples_reduced
])
print "Elements OUTSIDE hull (to include): {}/{}".format(
nr_elems_outside_hull, len(samples))
else:
inclusion_mask = np.array([
True if hull.find_simplex(sample) >= 0 else False
for sample in samples_reduced
])
# add samples (samples need to be np.array)
self.__data = np.concatenate((self.__data, samples[inclusion_mask]))
# =======================================
# 4. Recalculate hull with newly added points
# If memory exceeded: Perform unrefinement process -
# discharge sampling directions with lowest variance contribution
if self.dim_reduction > 0:
nr_comps = self.dim_reduction if len(
self.__data) <= self.max_size else self.dim_removal
if len(self.__data) > 150:
nr_comps = self.dim_removal - 1
basis, mean, var = ExtractMaxVarComponents(self.__data, nr_comps)
else:
# automatic dimension selection (based on containing certain variance)
basis, mean = ExtractSubspace(self.__data, 0.75)
cluster_reduced = ProjectOntoSubspace(self.__data, mean, basis)
print "Recuding dimension: {}->{}".format(
np.shape(self.__data)[1],
np.shape(cluster_reduced)[1])
hull = Delaunay(cluster_reduced)
# =======================================
# 5. Discharge samples inside hull
if not self.__inverted:
# select samples inside hull
cl_to_delete = np.array(
list(
set(range(0, len(cluster_reduced))) -
set(np.unique(hull.convex_hull))))
# set(range(len(data_hull))).difference(hull.convex_hull)
else:
cl_to_delete = np.array([])
# select samples on hull
if len(cluster_reduced) > self.max_size:
hull_indices = list(np.unique(hull.convex_hull))
if len(hull_indices) > 0:
nr_to_del = 5 if len(hull_indices) > 5 else 0
cl_to_delete = np.array(hull_indices[0:nr_to_del])
# print "Points building convex hull: {}".format(set(np.unique(hull.convex_hull)))
# print "To delete: {}".format(cl_to_delete)
if len(cl_to_delete[cl_to_delete < 0]) > 0:
print set(np.unique(hull.convex_hull))
log.warning("Index elements smaller than 0: {}".format(
cl_to_delete[cl_to_delete < 0]))
if self.__log:
self.log_intra_deleted.append(len(cl_to_delete))
self.log_cl_size_orig.append(len(self.__data))
print "Cleaning {} points from inside data".format(len(cl_to_delete))
# Remove points from inside hull
self.__data = np.delete(self.__data, cl_to_delete, axis=0)
# =======================================
# 6. KNN point removal: remove similar points
if self.knn_removal_thresh > 0:
max_removal = 10 if len(
self.__data) > self.knn_removal_thresh else 0
if max_removal > 0:
filter = KNFilter(self.__data, k=3, threshold=0.25)
tmp = filter.filter_x_samples(max_removal)
print "--- Removing {} knn points".format(
len(self.__data) - len(tmp))
self.__data = tmp
if self.__log:
self.log_cl_size_reduced.append(len(self.__data))
print "Cluster size: {}".format(len(self.__data))
def __reduce_after(self, metric='euclidean', reverse=True):
# TODO: clean samples with same pose but large variation (most likely erronomous)
if len(self.data) < self.__max_size:
return
# prevent drift - keep frontal poses
# keep at most 1/5 of the samples in the core to prevent drift
# - filter only other samples
if self.__prevent_drift:
# take all non-frontal images (any rotation > 10 deg)
m1 = abs(self.poses) > 10
non_frontal_mask = np.any(m1, axis=1)
frontal_mask = ~non_frontal_mask
non_frontal_indices = np.flatnonzero(non_frontal_mask)
frontal_indices = np.flatnonzero(frontal_mask)
# indices to filter
reduce_indices = non_frontal_indices
save_indices = frontal_indices
# take at minimum 4/5 non-frontal images
save_size = int(self.__max_size * 4. / 5.)
if len(non_frontal_indices) < save_size:
# print reduce_indices
# print save_indices
# print "Save size: ", save_size
# print "Set sizes: frontal: to add: {}, non-frontal: {}".format(save_size-len(non_frontal_indices), len(non_frontal_indices))
reduce_indices = np.concatenate(
(reduce_indices,
frontal_indices[0:(save_size -
len(non_frontal_indices))]))
save_indices = frontal_indices[save_size -
len(non_frontal_indices):]
# print "Samples to reduce: ", len(reduce_indices)
# print "Remaining save samples: ", len(save_indices)
# temporary data
data_tmp_save = self.data[save_indices]
pose_tmp_save = self.poses[save_indices]
data_tmp = self.data[reduce_indices]
pose_tmp = self.poses[reduce_indices]
# do filtering
dist = pairwise_distances(self.data_mean, data_tmp, metric=metric)
dist = dist[0]
if metric == 'euclidean':
dist = np.square(dist)
to_remove = len(self.data) - self.__max_size
indices = np.arange(0, len(data_tmp))
dist_sorted, indices_sorted = zip(
*sorted(zip(dist, indices), reverse=reverse))
indices_to_delete = indices_sorted[0:to_remove]
log.info(
'cl', "Removing {} non-frontal points".format(
len(indices_to_delete)))
data_tmp = np.delete(data_tmp, indices_to_delete, axis=0)
pose_tmp = np.delete(pose_tmp, indices_to_delete, axis=0)
# combine filtered with save data
self.data = np.concatenate((data_tmp, data_tmp_save))
self.poses = np.concatenate((pose_tmp, pose_tmp_save))
else:
# delete X samples which are most distant
dist = pairwise_distances(self.data_mean, self.data, metric=metric)
dist = dist[0]
if metric == 'euclidean':
dist = np.square(dist)
to_remove = len(self.data) - self.__max_size
indices = np.arange(0, len(self.data))
dist_sorted, indices_sorted = zip(
*sorted(zip(dist, indices), reverse=reverse))
indices_to_delete = indices_sorted[0:to_remove]
log.info('cl', "Removing {} points".format(len(indices_to_delete)))
# delete
self.data = np.delete(self.data, indices_to_delete, axis=0)
self.poses = np.delete(self.poses, indices_to_delete, axis=0)
def accumulate_samples(self,
tracking_id,
new_samples,
sample_weights=np.array([]),
sample_poses=np.array([])):
# check for set inconsistency
samples_ok = BaseMetaController.check_inter_sample_dist(
new_samples, metric='euclidean')
if not samples_ok:
log.severe("Identification set is inconsistent - disposing...")
# reset queue
self.sample_queue.pop(tracking_id, None)
self.sample_weight_queue.pop(tracking_id, None)
self.sample_pose_queue.pop(tracking_id, None)
return False, np.array([]), np.array([]), np.array([])
# generate placeholder weights
if sample_weights.size == 0:
# 5 of 10
sample_weights = np.repeat(5, len(new_samples))
assert len(sample_weights) == len(new_samples)
# add samples
if tracking_id not in self.sample_queue:
# initialize
self.sample_queue[tracking_id] = new_samples
self.sample_weight_queue[tracking_id] = sample_weights
self.sample_pose_queue[tracking_id] = sample_poses
else:
# append
self.sample_queue[tracking_id] = np.concatenate((self.sample_queue[tracking_id], new_samples))\
if self.sample_queue[tracking_id].size \
else new_samples
self.sample_weight_queue[tracking_id] = np.concatenate((self.sample_weight_queue[tracking_id], sample_weights))\
if self.sample_weight_queue[tracking_id].size \
else sample_weights
self.sample_pose_queue[tracking_id] = np.concatenate((self.sample_pose_queue[tracking_id], sample_poses))\
if self.sample_pose_queue[tracking_id].size \
else sample_poses
is_save_set = False
# if set has save sample or is long enough
if len(self.sample_queue[tracking_id]) >= self.min_sample_length:
if len(self.sample_queue[tracking_id]) >= self.save_sample_length\
or np.count_nonzero(self.sample_weight_queue[tracking_id] >= self.save_weight_thresh):
# check set consistency
samples_ok = BaseMetaController.check_inter_sample_dist(
self.sample_queue[tracking_id], metric='euclidean')
if samples_ok:
# set is save - allow identification
is_save_set = True
else:
# dispose all samples
self.sample_queue.pop(tracking_id, None)
self.sample_weight_queue.pop(tracking_id, None)
self.sample_pose_queue.pop(tracking_id, None)
log.severe("Set is inconsistent - disposing...")
# TODO: return whole set or only last?
current_samples = self.sample_queue.get(tracking_id, np.array([]))
current_weights = self.sample_weight_queue.get(tracking_id,
np.array([]))
current_poses = self.sample_pose_queue.get(tracking_id, np.array([]))
# not enough save samples - return what we have so far
return is_save_set, current_samples, current_weights, current_poses
def accumulate_samples(self,
user_id,
new_samples,
sample_weights=np.array([]),
sample_poses=np.array([])):
"""
:param user_id:
:param new_samples:
:param sample_weights:
:return:
array : save samples (save to integrate in any way)
bool : reset user
int : prediction of last section
float : confidence of last section prediction
"""
# check for set inconsistency
samples_ok = BaseMetaController.check_inter_sample_dist(
new_samples, metric='euclidean')
if not samples_ok:
# no return (queue is not filled up and thus we dont have a save section)
log.severe("Update set is inconsistent - disposing...")
# reset queue
self.sample_queue.pop(user_id, None)
self.sample_weight_queue.pop(user_id, None)
self.sample_pose_queue.pop(user_id, None)
return np.array([]), np.array([]), True, -1, 1.
# generate placeholder weights
if sample_weights.size == 0:
# 5 of 10
sample_weights = np.repeat(5, len(new_samples))
assert len(sample_weights) == len(new_samples)
# add samples
if user_id not in self.sample_queue:
# initialize
self.sample_queue[user_id] = new_samples
self.sample_weight_queue[user_id] = sample_weights
self.sample_pose_queue[user_id] = sample_poses
else:
# append
self.sample_queue[user_id] = np.concatenate((self.sample_queue[user_id], new_samples))\
if self.sample_queue[user_id].size \
else new_samples
self.sample_weight_queue[user_id] = np.concatenate((self.sample_weight_queue[user_id], sample_weights))\
if self.sample_weight_queue[user_id].size \
else sample_weights
self.sample_pose_queue[user_id] = np.concatenate((self.sample_pose_queue[user_id], sample_poses))\
if self.sample_pose_queue[user_id].size \
else sample_poses
target_class = -1
confidence = 1.
forward = np.array([])
forward_poses = np.array([])
reset_user = False
# do meta recognition
# check set for inconsistencies - return only save section
while len(self.sample_queue[user_id]) >= self.__queue_max_length:
sample_batch = self.sample_queue[user_id][0:self.
__queue_max_length]
weight_batch = self.sample_weight_queue[user_id][
0:self.__queue_max_length]
pose_batch = self.sample_pose_queue[user_id][0:self.
__queue_max_length]
# check set consistency
samples_ok = BaseMetaController.check_inter_sample_dist(
sample_batch, metric='euclidean')
# predict class
is_consistent, target_class, confidence = self.__p_multicl.predict_class(
sample_batch, sample_poses=pose_batch)
if samples_ok and is_consistent:
# add samples to forward
forward = np.concatenate((forward, self.sample_queue[user_id][0:self.__inclusion_range])) \
if forward.size \
else self.sample_queue[user_id][0:self.__inclusion_range]
forward_poses = np.concatenate((forward_poses, self.sample_pose_queue[user_id][0:self.__inclusion_range])) \
if forward_poses.size \
else self.sample_pose_queue[user_id][0:self.__inclusion_range]
# remove first x samples
self.sample_queue[user_id] = self.sample_queue[user_id][
self.__inclusion_range:]
self.sample_weight_queue[user_id] = self.sample_weight_queue[
user_id][self.__inclusion_range:]
self.sample_pose_queue[user_id] = self.sample_pose_queue[
user_id][self.__inclusion_range:]
else:
# dispose all samples! Whole queue!
self.sample_queue.pop(user_id, None)
self.sample_weight_queue.pop(user_id, None)
self.sample_pose_queue.pop(user_id, None)
log.severe("Set is inconsistent - disposing...")
reset_user = True
break
# predict user if not enough samples
if not forward.size and reset_user is False:
is_consistent, target_class, confidence = self.__p_multicl.predict_class(
self.sample_queue[user_id],
sample_poses=self.sample_pose_queue[user_id])
print "Not enough to forward but predict...", is_consistent, target_class, confidence
return forward, forward_poses, reset_user, target_class, confidence
def __init__(self, server, conn, handle):
# receive tracking id
tracking_id = server.receive_uint(conn)
# receive images
images = server.receive_image_batch_squared_same_size(
conn, switch_rgb_bgr=True)
# get sample poses
sample_poses = []
for x in range(0, len(images)):
pitch = server.receive_char(conn)
yaw = server.receive_char(conn)
sample_poses.append([pitch, yaw])
sample_poses = np.array(sample_poses)
# generate embedding (from rgb images)
embeddings = server.embedding_gen.get_embeddings(rgb_images=images,
align=False)
if not embeddings.any():
r.Error(server, conn, "Could not generate face embeddings.")
return
# accumulate samples
is_save_set, current_samples, _weights_placeholder, current_poses = \
server.classifier.id_controller.accumulate_samples(
tracking_id, new_samples=embeddings, sample_poses=sample_poses
)
log.info('server', "tracking id: {}".format(tracking_id))
if len(current_samples) == 0:
# queue has just been resetted
r.Error(
server, conn,
"Samples are inconsistent - starting accumulation again...")
return
# predict class similarities
new_class_guaranteed = server.classifier.is_guaranteed_new_class(
current_samples)
# at least 3 samples needed to generate classifier
if new_class_guaranteed and len(current_samples) > 2:
id_pred = -1
confidence = 100
is_consistent = True # yes - inter-sample distance checked
is_save_set = True
else:
# do meta recognition and predict the user id from the cls scores
is_consistent, id_pred, confidence = server.classifier.predict_class(
current_samples, sample_poses=current_poses)
# convert to integer
confidence = int(confidence * 100.0)
# get user nice name
user_name = server.user_db.get_name_from_id(id_pred)
print ".... is_save: {}, is_consistent: {}, id_pred: {}, confidence: {}".format(
is_save_set, is_consistent, id_pred, confidence)
if user_name is None:
user_name = "unnamed"
# save images
if DEBUG_IMAGES:
for i in images:
# save from RGB order
scipy.misc.imsave(
"identification_{}.png".format(current_milli_time()), i)
if is_save_set:
# SAVE SET - TAKE ACTION
profile_picture = None
if is_consistent:
# new identity
if id_pred < 0:
# unknown user
print "--- creating new user"
log.info('db', "Creating new User")
user_id = server.user_db.create_new_user("a_user")
server.user_db.print_users()
# add classifier
server.classifier.init_new_class(
user_id, current_samples, sample_poses=current_poses)
id_pred = user_id
else:
# for s in current_samples:
# print "s: {:.2f}".format(s[0])
# add data for training and return identification
# add to data model
server.classifier.data_controller.add_samples(
user_id=id_pred,
new_samples=current_samples,
new_poses=current_poses)
# add to classifier training queue
server.classifier.add_training_data(
id_pred, current_samples)
# get profile picture
profile_picture = server.user_db.get_profile_picture(
id_pred)
# cleanup
server.classifier.id_controller.drop_samples(tracking_id)
# valid identification
log.info(
'server',
"User identification complete: {} [ID], {} [Username]".
format(id_pred, user_name))
r.Identification(server,
conn,
int(id_pred),
user_name,
confidence=confidence,
profile_picture=profile_picture)
else:
# inconsistent prediction - samples might be bad. dump and take new samples
server.classifier.id_controller.drop_samples(tracking_id)
r.Error(
server, conn,
"Samples are inconsistent - starting accumulation again..."
)
return
# TODO: is feedback useful here?
else:
# UNSAVE SET - WAIT TILL SAVE SET IS ACCUMULATED
# identification progress in percent
id_progress = len(current_samples) / float(
server.classifier.id_controller.save_sample_length)
id_progress = int(id_progress * 100)
# return prediction and confidence - but no identification
r.PredictionFeedback(server,
conn,
int(id_pred),
user_name,
confidence=confidence,
progress=id_progress)
def __init__(self, server, conn, handle):
# receive user id
user_id = server.receive_uint(conn)
log.info('server',
'User Update (Aligned, Robust) for ID {}'.format(user_id))
# receive images
images = server.receive_image_batch_squared_same_size(
conn, switch_rgb_bgr=True)
# save images
if DEBUG_IMAGES:
for i in images:
# save from RGB order
scipy.misc.imsave("update_{}.png".format(current_milli_time()),
i)
# get sample poses
sample_poses = []
for x in range(0, len(images)):
pitch = server.receive_char(conn)
yaw = server.receive_char(conn)
sample_poses.append([pitch, yaw])
sample_poses = np.array(sample_poses)
# generate embedding
embeddings = server.embedding_gen.get_embeddings(rgb_images=images,
align=False)
if not embeddings.any():
r.Error(server, conn, "Could not generate face embeddings.")
return
# TODO: calculate weights
weights = np.repeat(10, len(images))
# accumulate samples - check for inconsistencies
verified_data, verified_poses, reset_user, id_pred, confidence = server.classifier.update_controller.accumulate_samples(
user_id,
embeddings,
sample_weights=weights,
sample_poses=sample_poses)
log.info(
'cl',
"verified_data (len: {}), reset_user: {}: ID {}, conf {}".format(
len(verified_data), reset_user, id_pred, confidence))
# forward save part of data
if verified_data.size:
# for s in embeddings:
# print "new: {:.8f}".format(s[0])
# print "------------------"
# for s in verified_data:
# print "s: {:.5f}".format(s[0])
# add to data model
server.classifier.data_controller.add_samples(
user_id=user_id,
new_samples=verified_data,
new_poses=verified_poses)
# add to classifier training queue
server.classifier.add_training_data(user_id, verified_data)
# reset user if queue has become inconsistent or wrong user is predicted
if reset_user:
log.severe("USER VERIFICATION FAILED - FORCE REIDENTIFICATION")
r.Reidentification(server, conn)
return
# return prediction feedback
user_name = server.user_db.get_name_from_id(id_pred)
if user_name is None:
user_name = "unnamed"
r.PredictionFeedback(server,
conn,
id_pred,
user_name,
confidence=int(confidence * 100.0))
def predict(self, samples):
"""
Prediction cases:
- Only target class is identified with ratio X (high): Class
- Target and other class is identified with ration X (high) and Y (small): Class with small confusion
- Multiple classes are identified with small ratios Ys: Novelty
- No classes identified: Novelty
:param samples:
:return: Class ID, -1 (Novelty), None invalid samples (multiple detections)
"""
# no classifiers yet, predict novelty
if not self.classifiers:
# 100% confidence
self.__decision_function = np.array([len(samples)]), np.array([-1])
return -1
predictions, class_ids = self.__predict(samples)
if len(predictions) == 0:
# no class in reach - classify as novel class
self.__decision_function = np.array([len(samples)]), np.array([-1])
return -1
# calc nr of positive class detections
cls_scores = (predictions > 0).sum(axis=1)
self.__decision_function = cls_scores, class_ids
nr_samples = len(samples)
self.__decision_nr_samples = nr_samples
log.info(
'cl',
"Classifier scores: {} | max: {}".format(cls_scores, nr_samples))
# no classes detected at all - novelty
if len(cls_scores[cls_scores <= self.__novelty_thresh *
nr_samples]) == len(cls_scores):
return -1
identification_mask = cls_scores >= self.__class_thresh * nr_samples
ids = class_ids[identification_mask]
if len(ids) > 0:
# multiple possible detection - invalid samples
if len(ids) > 1:
# use average to-class-distance to select best choice
mean_dist_cosine = []
mean_dist_euclidean = []
# todo: mean dist or mean dist to cluster mean
for class_id in ids:
mean_dist_cosine.append(
self.classifiers[class_id].mean_dist(samples))
mean_dist_euclidean.append(
self.classifiers[class_id].mean_dist(
samples, 'euclidean'))
id_index_cosine = mean_dist_cosine.index(min(mean_dist_cosine))
id_index_euclidean = mean_dist_euclidean.index(
min(mean_dist_euclidean))
log.severe("Samples are inambiguous. Classes: {}".format(ids))
log.severe("IDCOS: {} | meandist cosine: {}".format(
int(ids[id_index_cosine]), mean_dist_cosine))
log.severe("IDEUC: {} | meandist euclidean: {}".format(
int(ids[id_index_euclidean]), mean_dist_euclidean))
for class_id in ids:
print self.classifiers[class_id].class_mean_dist(
samples, 'cosine')
mean_dist_cosine = np.array(mean_dist_cosine)
if np.sum(
(mean_dist_cosine - min(mean_dist_cosine)) < 0.05) > 1:
log.severe(
"SAMPLES DISCARGED: Average distance to data inambiguous"
)
return None
return int(ids[id_index_cosine])
# return None
# single person identified - return id
return int(ids[0])
else:
# samples unclear
return None