def pre_identification_helper(images, probe_id, settings, hash_config_path,
callback_code):
logger.info(
'Running verification for user - %s, with given parameters - %s' %
(settings.get('userID'), settings))
if AlgorithmsDataStore.instance().exists(key=REDIS_JOB_RESULTS_ERROR %
callback_code):
logger.info(
'Job interrupted because of job_results_error key existence.')
return None
temp_image_path = tempfile.mkdtemp(dir=APP_ROOT)
try:
image_paths = save_images(images, temp_image_path)
# Store photos for test purposes
store_test_photo_helper(APP_ROOT, image_paths)
settings.update({'data': image_paths})
settings.update({
'general_data': {
'data_path': temp_image_path,
'hash_config_path': hash_config_path,
'probe_id': probe_id
}
})
return settings
except:
return None
def final_helper(temp_image_path, probe_id, error, callback_code, result,
ai_response_type, try_type, ai_code):
shutil.rmtree(temp_image_path)
if error is not None:
retries_count = AlgorithmsDataStore.instance().decrement_int_value(
REDiS_TRAINING_RETRIES_COUNT_KEY % probe_id)
if retries_count == 0:
AlgorithmsDataStore.instance().delete_data(
key=REDiS_TRAINING_RETRIES_COUNT_KEY % probe_id)
logger.debug('Maximum training attempts reached...')
result = False
ai_response_type.update(
dict(status=TRAINING_MAX_RETRIES_STATUS,
message=TRAINING_MAX_RETRIES_MESSAGE))
# _tell_ai_training_results(result, ai_response_type, try_type, ai_code)
else:
AlgorithmsDataStore.instance().store_data(
key=REDIS_UPDATE_TRAINING_KEY % probe_id, error=error)
result = dict(result=False, error=error)
AlgorithmsDataStore.instance().store_data(key=REDIS_PROBE_RESULT_KEY %
callback_code,
result=result)
logger.info('Job was finished with internal algorithm error %s ' %
error)
else:
AlgorithmsDataStore.instance().delete_data(
key=REDiS_TRAINING_RETRIES_COUNT_KEY % probe_id)
AlgorithmsDataStore.instance().store_data(key=REDIS_PROBE_RESULT_KEY %
callback_code,
result=result)
tell_ai_training_results(result, ai_response_type, try_type, ai_code)
def pre_verification_helper(image, settings, probe_id, callback_code):
logger.info(
'Running verification for user - %s, with given parameters - %s' %
(settings.get('userID'), settings))
if RedisStorage.persistence_instance().exists(key=REDIS_JOB_RESULTS_ERROR %
callback_code):
logger.info(
'Job interrupted because of job_results_error key existence.')
return
database = get_algo_db(probe_id=probe_id)
temp_image_path = tempfile.mkdtemp(dir=APP_ROOT)
try:
temp_image = save_image(image, temp_image_path)
# Store photos for test purposes
backup_path = store_test_photo_helper(
APP_ROOT, [temp_image],
"candidates_{}".format(settings.get('userID')))
settings.update({
'data': temp_image,
'database': database,
'temp_image_path': temp_image_path,
'backup_image_path': backup_path
})
return settings
except Exception as e:
logger.exception(e)
return None
def importSources(self, source):
logger.debug("Database loading started...")
self._etalon = ClusterL0Estimation.importDatabase(
source.get('data', dict()))
# self._etalon = SelfGraphEstimation.importDatabase(source.get('data', dict()))
self._prob = source.get('threshold', 100)
logger.info('Dynamic threshold: %f' % self._prob)
logger.debug("Database loading finished.")
def importSettings(self, settings):
if len(settings.keys()) > 0:
logger.info("Settings loading started...")
self.kodsettings.importSettings(settings['KODSettings'])
self.kodsettings.dump()
if self._cascadeROI is None:
self._cascadeROI = OptimalROIDetectorSAoS()
if self._eyeROI is None:
self._eyeROI = CascadesDetectionInterface(loadScript("main_haarcascade_eyes_union.json", True))
logger.info("Settings loading finished.")
return True
return False
def apply(self, data):
"""
:param data: dictionary = {
'temp_image_path': path to the temporary image storage,
'probe_id': probe identifier,
'error': error message,
'callback_code': code of callback function,
'result': status of result
'ai_response_type': type of AI response,
'try_type': type of try
'ai_code': AI response code
}
:return:
"""
temp_image_path = data['temp_image_path']
error = data['error']
probe_id = data['probe_id']
ai_response_type = data['ai_response_type']
callback_code = data['callback_code']
result = data['result']
try_type = data['try_type']
ai_code = data['ai_code']
shutil.rmtree(temp_image_path)
if error is not None:
retries_count = AlgorithmsDataStore.instance().decrement_int_value(
REDiS_TRAINING_RETRIES_COUNT_KEY % probe_id)
if retries_count == 0:
AlgorithmsDataStore.instance().delete_data(
key=REDiS_TRAINING_RETRIES_COUNT_KEY % probe_id)
logger.debug('Maximum training attempts reached...')
result = False
ai_response_type.update({
'status': TRAINING_MAX_RETRIES_STATUS,
'message': TRAINING_MAX_RETRIES_MESSAGE
})
else:
AlgorithmsDataStore.instance().store_data(
key=REDIS_UPDATE_TRAINING_KEY % probe_id, error=error)
result = dict(result=False, error=error)
AlgorithmsDataStore.instance().store_data(
key=REDIS_PROBE_RESULT_KEY % callback_code, result=result)
logger.info('Job was finished with internal algorithm error %s ' %
error)
else:
AlgorithmsDataStore.instance().delete_data(
key=REDiS_TRAINING_RETRIES_COUNT_KEY % probe_id)
AlgorithmsDataStore.instance().store_data(
key=REDIS_PROBE_RESULT_KEY % callback_code, result=result)
tell_ai_training_results(result, ai_response_type, try_type, ai_code)
return None
def _detect(self, data, detector):
# ROI detection
rect = self._eyeROI.detect(data['roi'])[1]
if len(rect) <= 0 or len(rect[0]) <= 0:
logger.info("Eye ROI wasn't found.")
self._last_error = "Eye ROI wasn't found."
return False
# ROI cutting
rect = rect[0]
lefteye = (rect[0] + rect[3], rect[1] + rect[3] / 2)
righteye = (rect[0] + rect[2] - rect[3], rect[1] + rect[3] / 2)
centereye = (lefteye[0] + (righteye[0] - lefteye[0]) / 2, lefteye[1] + (righteye[1] - lefteye[1]) / 2)
centernose = (lefteye[0] + (righteye[0] - lefteye[0]) / 2, rect[1] + 2 * rect[3])
centermouth = (centernose[0], centernose[1] + rect[3])
leftmouth = (lefteye[0], centermouth[1])
rightmouth = (righteye[0], centermouth[1])
centers = [lefteye, righteye, centereye, centernose, leftmouth, rightmouth]
# self.filter_keypoints(data)
clusters = KMeans(data['keypoints'], 0, centers)
data['true_clusters'] = clusters
descriptors = []
centers = []
weights = []
keydescriptors = []
active_clusters = 0
for cluster in clusters:
desc = detector.compute(data['roi'], cluster['items'])
curr_cluster = desc['descriptors']
descriptors.append(curr_cluster)
centers.append(cluster['center'])
if curr_cluster is not None and len(curr_cluster) > 0:
active_clusters += 1
pairs = []
sum = 0
for index, item in enumerate(cluster['items']):
# pairs.append((desc['descriptors'][index], item.response))
sum += item.response
weights.append((pairs, sum))
cluster_keypoints = []
for index, item in enumerate(cluster['items']):
cluster_keypoints.append((item, desc['descriptors'][index]))
keydescriptors.append(cluster_keypoints)
data['clusters'] = descriptors
data['centers'] = centers
data['weights'] = weights
data['key_desc'] = keydescriptors
if active_clusters < len(centers) - 2:
logger.info("Number of clusters are insufficient for the recognition.")
self._last_error = "Number of clusters are insufficient for the recognition."
return False
return True
def delete_data(table_class_name, object_ids):
logger.info('Delete records for table class - %s, with object_ids - %s' %
(table_class_name, object_ids))
try:
MySQLDataStoreInterface.delete_multiple_data(
table_name=table_class_name, object_ids=object_ids)
records = dict(status=True)
except Exception as e:
logger.exception(e)
records = dict(error=str(e), status=False)
logger.info('Deleted records for table class - %s, with object_ids - %s' %
(table_class_name, object_ids))
return records
def select_records_by_ids(table_class_name, object_ids, flat_result=False):
logger.info('Getting records for table class - %s, with object_ids - %s' %
(table_class_name, object_ids))
try:
records = MySQLDataStoreInterface.select_data_by_ids(
table_name=table_class_name,
object_ids=object_ids,
flat_result=flat_result)
logger.debug('Data: %s' % records)
except Exception as e:
logger.exception(e)
records = dict(error=str(e))
logger.info('Got records for table class - %s, with object_ids - %s' %
(table_class_name, object_ids))
return records
def create_records(table_class_name, values, update=False):
logger.info(
'Store records for table class - %s, values - %s (with updating - %s)'
% (table_class_name, values, update))
try:
MySQLDataStoreInterface.create_multiple_records(
table_name=table_class_name, values=values, update=update)
records = dict(status=True)
except Exception as e:
logger.exception(e)
records = dict(error=str(e), status=False)
logger.info(
'Stored records for table class - %s, with object_ids - %s (with updating - %s)'
% (table_class_name, values, update))
return records
def pre_training_helper(images, probe_id, settings, callback_code, try_type,
ai_code):
logger.info('Running training for user - %s, with given parameters - %s' %
(settings.get('userID'), settings))
ai_response_type = dict()
try:
logger.info('Telling AI that we are starting training with code - %s' %
ai_code)
ai_response_type.update({
'status': TRAINING_STARTED_STATUS,
'message': TRAINING_STARTED_MESSAGE
})
ai_response_sender(ai_code, ai_response_type)
except Exception as e:
logger.error('Failed to build rest request to AI - %s' % str(e))
logger.exception(e)
ai_response_type.update({
'status': TRAINING_SUCCESS_STATUS,
'message': TRAINING_SUCCESS_MESSAGE
})
result = False
error = None
if AlgorithmsDataStore.instance().exists(key=REDIS_UPDATE_TRAINING_KEY %
probe_id):
settings.update({'database': get_algo_db(probe_id=probe_id)})
AlgorithmsDataStore.instance().delete_data(
key=REDIS_UPDATE_TRAINING_KEY % probe_id)
temp_image_path = tempfile.mkdtemp(dir=APP_ROOT)
try:
image_paths = save_images(images, temp_image_path)
# Store photos for test purposes
store_test_photo_helper(APP_ROOT, image_paths)
settings.update({'data': image_paths})
settings.update({
'general_data': {
'data_path': temp_image_path,
'ai_code': ai_code,
'try_type': try_type,
'probe_id': probe_id
}
})
return settings
except:
final_helper(temp_image_path, probe_id, error, callback_code, result,
ai_response_type, try_type, ai_code)
return None
def result_handling(result,
probe_id,
temp_image_path,
settings,
callback_code,
error=None):
if error is not None or RedisStorage.persistence_instance().exists(
key=REDIS_JOB_RESULTS_ERROR % callback_code):
if not RedisStorage.persistence_instance().exists(
key=REDIS_JOB_RESULTS_ERROR % callback_code):
result = dict(verified=False, error=error)
RedisStorage.persistence_instance().store_data(
key=REDIS_JOB_RESULTS_ERROR % callback_code,
ex=300,
result=result)
store_verification_results(result=result,
callback_code=callback_code,
probe_id=probe_id)
if error is not None:
logger.info('Job was finished with internal algorithm error %s ' %
error)
else:
logger.info(
'Job was not stored because of job_results_error key existence.'
)
else:
RedisStorage.persistence_instance().append_value_to_list(
key=REDIS_PARTIAL_RESULTS_KEY % callback_code, value=result)
results_counter = RedisStorage.persistence_instance(
).decrement_int_value(REDIS_RESULTS_COUNTER_KEY % callback_code)
if results_counter <= 0:
gathered_results = RedisStorage.persistence_instance(
).get_stored_list(REDIS_PARTIAL_RESULTS_KEY % callback_code)
logger.debug('All gathered results for verification job - %s' %
gathered_results)
if results_counter < 0:
logger.exception(
'Results count is less than 0, check worker jobs consistency!'
)
result = dict(verified=False)
else:
true_count = float(gathered_results.count('True'))
result = dict(verified=(
(true_count / len(gathered_results)) * 100) >= 50)
store_verification_results(result=result,
callback_code=callback_code,
probe_id=probe_id)
shutil.rmtree(temp_image_path)
logger.info('Verification finished for user - %s, with result - %s' %
(settings.get('userID'), result))
def apply(self, data):
"""
:param data: dictionary = {
'images': training image set,
'probe_id': prove identifier,
'settings': general settings dictionary,
'callback_code': code of callback function,
'try_type': type of try request,
'ai_code': AI response code,
'temp_data_dir': Path to the temporary storage
}
:return:
"""
images = data['images']
settings = data['settings']
ai_code = data['ai_code']
probe_id = data['probe_id']
try_type = data['try_type']
temp_data_dir = data['temp_data_dir']
logger.info(
'Running training for user - %s, with given parameters - %s' %
(settings.get('userID'), settings))
ai_response_type = dict()
try:
logger.info(
'Telling AI that we are starting training with code - %s' %
ai_code)
ai_response_type.update({
'status': TRAINING_STARTED_STATUS,
'message': TRAINING_STARTED_MESSAGE
})
ai_response_sender(ai_code, ai_response_type)
except Exception as e:
# TODO: Write Error handler
logger.error('Failed to build rest request to AI - %s' % str(e))
logger.exception(e)
ai_response_type.update({
'status': TRAINING_SUCCESS_STATUS,
'message': TRAINING_SUCCESS_MESSAGE
})
result = False
if AlgorithmsDataStore.instance().exists(
key=REDIS_UPDATE_TRAINING_KEY % probe_id):
settings.update({'database': get_algo_db(probe_id=probe_id)})
AlgorithmsDataStore.instance().delete_data(
key=REDIS_UPDATE_TRAINING_KEY % probe_id)
temp_image_path = tempfile.mkdtemp(dir=temp_data_dir)
try:
image_paths = save_images(images, temp_image_path)
# Store photos for test purposes
store_test_photo_helper(temp_data_dir, image_paths,
"train_{}".format(settings.get('userID')))
settings.update({'data': image_paths})
settings.update({
'general_data': {
'data_path': temp_image_path,
'ai_code': ai_code,
'try_type': try_type,
'probe_id': probe_id
}
})
return settings
except Exception as error:
end_data = data.copy()
del end_data['images']
del end_data['settings']
end_data.update({
'temp_image_path': temp_image_path,
'error': error,
'result': result,
'ai_response_type': ai_response_type
})
return self._stages.get(FINAL_TRAINING_STAGE).apply(end_data)