def main():
if not mlboard:
print_fun("Skipped: no mlboard detected")
return
parser = argparse.ArgumentParser()
add_common_args(parser)
parser.add_argument(
'model_name',
help='Exported model name.',
type=str,
)
parser.add_argument(
'model_version',
help='Exported model version.',
type=str,
)
args = parser.parse_args()
utils.print_fun('Uploading model...')
mlboard.model_upload(args.model_name, args.model_version,
args.classifiers_dir)
update_task_info({
'model_reference':
catalog_ref(args.model_name, 'mlmodel', args.model_version)
})
utils.print_fun("New model uploaded as '%s', version '%s'." %
(args.model_name, args.model_version))
def listen(self):
while True:
try:
command, data = pickle.loads(self.socket.recv())
# test, remove
if command == "error":
raise ValueError("test error")
return command, data
except Exception as e:
utils.print_fun("listener exception: %s" % e)
self.result(e)
def load_dataset(dataset_dir):
ds = dataset.get_dataset(dataset_dir)
size = 0
for cls in ds:
size += len(cls.image_paths)
imgs_original = [''] * size
imgs = [''] * (size * 4)
imgs_cls = [''] * size
issame = np.zeros([size * 2], dtype=np.bool)
i = 0
for cls in ds:
for j, path in enumerate(cls.image_paths):
img_path = path
imgs_original[i] = img_path
imgs[i + i] = img_path
imgs[i + i + size * 2] = img_path + ':flip'
imgs_cls[i] = cls.name
i += 1
# Expand dataset with random pairs
utils.print_fun('Generating pairs for dataset...')
for i in range(len(imgs_original) * 2):
if i % 2 == 0:
target = lambda x: not x
else:
target = lambda x: x
j = np.random.randint(0, len(imgs_original) * 2)
if target(imgs_cls[i % size] != imgs_cls[j % size]):
shift = 1
j1 = j - shift if j > 0 else 0
j2 = j + shift if j < len(imgs_original) * 2 else len(
imgs_original) - 1
while target(imgs_cls[i % size] != imgs_cls[j1 % size]) and target(
imgs_cls[i % size] != imgs_cls[j2 % size]):
j1 = j - shift if j1 > 0 else 0
j2 = j + shift if j2 < len(imgs_original) * 2 else len(
imgs_original) - 1
shift += 1
if target(imgs_cls[i % size] == imgs_cls[j1 % size]):
j = j1
else:
j = j2
if j >= size:
imgs[i + i + 1] = imgs_original[j % size] + ':flip'
else:
imgs[i + i + 1] = imgs_original[j]
issame[i] = imgs_cls[i % size] == imgs_cls[j % size]
utils.print_fun('Done.')
return imgs, issame
def notify(**kwargs):
if notifier_disabled:
return
if notifier is None:
global not_init_say
if not not_init_say:
utils.print_fun("=== Notifications aren't initialized ===")
not_init_say = True
return
if not kwargs.get('name') and not kwargs.get('text'):
kwargs['name'] = '--Somebody--'
notifier.notify(**kwargs)
def extract(self):
try:
utils.print_fun("Downloading from %s" % self.url)
arc = io.BytesIO(urlopen(self.url).read())
except Exception as e:
return RuntimeError("Download archive error: %s" % e)
try:
utils.print_fun("Extracting...")
tar = tarfile.open(fileobj=arc, mode="r:*")
tar.extractall(path=self.dest)
tar.close()
except Exception as e:
return RuntimeError("Extract archive error: %s" % e)
return None
def call(self, command, data=None):
if self.socket is None:
# https://stackoverflow.com/questions/44273941/python-script-not-terminating-after-timeout-in-zmq-recv
nIOthreads = 2
self.context = zmq.Context(nIOthreads)
self.socket = self.context.socket(zmq.REQ)
self.socket.setsockopt(zmq.LINGER, 0)
self.socket.setsockopt(zmq.AFFINITY, 1)
self.socket.setsockopt(zmq.RCVTIMEO, 2000)
self.socket.connect('tcp://127.0.0.1:%d' % self.port)
self.socket.send(pickle.dumps((command, data)))
try:
err = pickle.loads(self.socket.recv())
except zmq.error.Again:
err = ValueError("no response, is server available?")
utils.print_fun(err)
self.socket.close()
self.context.term()
self.socket = None
self.context = None
return err
def notify(**kwargs):
override_text = kwargs.get('text')
if not override_text:
msg_strings = ['%s has been detected' % kwargs['name']]
else:
msg_strings = ['%s' % kwargs['text']]
if 'position' in kwargs:
msg_strings.append('Position: %s' % kwargs['position'])
if 'company' in kwargs:
msg_strings.append('Company: %s' % kwargs['company'])
if 'url' in kwargs:
msg_strings.append('URL: %s' % kwargs['url'])
if 'image' in kwargs:
msg_strings.append('[IMAGE]')
if 'action_options' in kwargs:
msg_strings.append('[OPTIONS]')
msg_strings.extend([' - ' + o for o in kwargs['action_options']])
if 'system' in kwargs:
msg_strings.append('System: %s' % kwargs['system'])
sl = max([len(s) for s in msg_strings])
utils.print_fun('=' * sl)
utils.print_fun('\r\n'.join(msg_strings))
utils.print_fun('=' * sl)
def main():
parser = argparse.ArgumentParser()
add_aligner_args(parser)
add_common_args(parser)
add_classifier_args(parser)
add_bg_remove_args(parser)
utils.add_normalization_args(parser)
parser.add_argument(
'--skip_align',
help=
'Skip alignment for source images from input dir, only calculate embeddings and train classifiers.',
action='store_true',
)
parser.add_argument(
'--skip_train',
help=
'Skip calculating embeddings and training, only alignment for source images from input dir.',
action='store_true',
)
parser.add_argument(
'--align_images_limit',
type=int,
help='Set limit for processed images in alignment.',
default=None,
)
parser.add_argument(
'--model_name',
help='Exported model name.',
type=str,
default=None,
)
parser.add_argument(
'--model_version',
help='Exported model version.',
type=str,
default=None,
)
parser.add_argument(
'--complementary',
help='Complementary align and training.',
action='store_true',
)
parser.add_argument(
'--best_threshold',
help='Find best threshold.',
action='store_true',
)
args = parser.parse_args()
if not args.skip_align:
al = aligner_args(args)
al.align(args.align_images_limit)
if not args.skip_train:
clf = classifiers_args(args)
clf.train()
from app.control.client import Client
cl = Client()
cl.call('reload_classifiers')
if args.model_name is not None and args.model_version is not None:
from app.mlboard import mlboard, update_task_info, catalog_ref
utils.print_fun('Uploading model...')
model_version = args.model_version
if args.device not in args.model_version:
model_version = args.model_version + '-%s' % args.device
mlboard.model_upload(args.model_name, model_version,
args.classifiers_dir)
update_task_info({
'model_reference':
catalog_ref(args.model_name, 'mlmodel', args.model_version)
})
utils.print_fun("New model uploaded as '%s', version '%s'." %
(args.model_name, args.model_version))
def align(self, images_limit=None):
if self.complementary_align:
utils.print_fun(
'Complementary align %simages to %s' %
("clarified " if self.clarified else "", self.aligned_dir))
else:
utils.print_fun('Align images to %s' % self.aligned_dir)
aligned_dir = os.path.expanduser(self.aligned_dir)
bounding_boxes_filename = os.path.join(aligned_dir,
'bounding_boxes.txt')
align_filename = os.path.join(aligned_dir, 'align.pkl')
align_data_args = {
"min_face_size": self.min_face_size,
"image_size": self.image_size,
"margin": self.margin,
# used for dataset alignment and do not used for clarified alignment
# "bg_remove_path": self.bg_remove_path,
}
align_data = {}
if os.path.isfile(align_filename):
utils.print_fun("Check previous align data")
with open(align_filename, 'rb') as infile:
(align_data_args_loaded,
align_data_loaded) = pickle.load(infile)
if align_data_args == align_data_args_loaded:
utils.print_fun("Loaded data about %d aligned classes" %
len(align_data_loaded))
align_data = align_data_loaded
else:
utils.print_fun(
"Previous align data is for another arguments, deleting existing data"
)
shutil.rmtree(aligned_dir, ignore_errors=True)
if not os.path.isdir(aligned_dir):
utils.print_fun("Creating output dir")
os.makedirs(aligned_dir)
# Store some git revision info in a text file in the log directory
src_path, _ = os.path.split(os.path.realpath(__file__))
# facenet.store_revision_info(src_path, output_dir, ' '.join(sys.argv))
loaded_dataset = dataset.get_dataset(self.input_dir)
loaded_dataset_meta = dataset.get_meta(loaded_dataset)
utils.print_fun('Creating networks and loading parameters')
# Load driver
self._load_driver()
bg_rm_drv = bg_remove.get_driver(self.bg_remove_path)
bounding_boxes_contents = ""
# clear not actual previous aligned stored data
if not self.complementary_align and len(align_data) > 0:
stored_classes = []
for cls in loaded_dataset:
stored_classes.append(cls.name)
for adcl in list(align_data):
if adcl not in stored_classes:
del align_data[adcl]
nrof_images_total = 0
nrof_images_skipped = 0
nrof_images_cached = 0
nrof_successfully_aligned = 0
nrof_has_meta = 0
for cls in loaded_dataset:
output_class_dir = os.path.join(aligned_dir, cls.name)
output_class_dir_created = False
# meta_file = None
aligned_class_images = []
if cls.name in align_data:
align_data_class = align_data[cls.name]
else:
align_data_class = {}
for image_path in cls.image_paths:
# if os.path.basename(image_path) == dataset.META_FILENAME:
# meta_file = image_path
# continue
nrof_images_total += 1
nrof_images_skipped += 1
filename = os.path.splitext(os.path.split(image_path)[1])[0]
output_filename = os.path.join(output_class_dir,
filename + '.png')
if not os.path.exists(output_filename):
try:
img = cv2.imread(image_path,
cv2.IMREAD_COLOR).astype(np.float32)
except Exception as e:
error_message = '{}: {}'.format(image_path, e)
utils.print_fun('ERROR: %s' % error_message)
continue
img_hash = hashlib.sha1(img.tostring()).hexdigest()
if image_path in align_data_class:
if 'hash' in align_data_class[image_path]:
if align_data_class[image_path][
'hash'] == img_hash:
all_aligned_exists = True
if 'aligned' in align_data_class[image_path]:
for a in align_data_class[image_path][
'aligned']:
if not os.path.isfile(a):
all_aligned_exists = False
break
if all_aligned_exists:
if 'aligned' in align_data_class[
image_path]:
aligned_class_images.extend(
list(align_data_class[image_path]
['aligned'].keys()))
for _, a in enumerate(
align_data_class[image_path]
['aligned']):
b = align_data_class[image_path][
'aligned'][a]
bounding_boxes_contents += \
'%s %d %d %d %d cached\n' % (a, b[0], b[1], b[2], b[3])
nrof_images_skipped -= 1
else:
bounding_boxes_contents += \
'%s ERROR no aligned cached\n' % image_path
nrof_images_cached += 1
continue
align_data_class[image_path] = {
'hash': hashlib.sha1(img.tostring()).hexdigest()
}
utils.print_fun(image_path)
if len(img.shape) <= 2:
utils.print_fun(
'WARNING: Unable to align "%s", shape %s' %
(image_path, img.shape))
bounding_boxes_contents += '%s ERROR invalid shape\n' % image_path
continue
if self.clarified:
# get clarified image as is and make one with aligned size
bounding_boxes = np.stack(
[[0, 0, img.shape[1], img.shape[0]]])
face_crop_margin = 0
else:
# detect faces previously with bg_remove if set, if not found, try to detect w/o bg_remove
if bg_rm_drv is not None:
img = bg_rm_drv.apply_mask(img)
bounding_boxes = None
if bg_rm_drv is not None:
img_masked = bg_rm_drv.apply_mask(img)
bounding_boxes = self._get_boxes(
image_path, img_masked)
if bounding_boxes is None:
utils.print_fun(
'WARNING: no faces on image with removed bg, trying without bg removing'
)
if bounding_boxes is None or bg_rm_drv is not None:
bounding_boxes = self._get_boxes(image_path, img)
if bounding_boxes is None:
bounding_boxes_contents += '%s ERROR no faces detected\n' % image_path
continue
face_crop_margin = self.margin
imgs = images.get_images(
img,
bounding_boxes,
face_crop_size=self.image_size,
face_crop_margin=face_crop_margin,
normalization=None,
)
align_data_class[image_path]['aligned'] = {}
for i, cropped in enumerate(imgs):
nrof_successfully_aligned += 1
bb = bounding_boxes[i]
filename_base, file_extension = os.path.splitext(
output_filename)
output_filename_n = "{}_{}{}".format(
filename_base, i, file_extension)
bounding_boxes_contents += '%s %d %d %d %d\n' % (
output_filename_n, bb[0], bb[1], bb[2], bb[3])
if not output_class_dir_created:
output_class_dir_created = True
if not os.path.exists(output_class_dir):
os.makedirs(output_class_dir)
cv2.imwrite(output_filename_n, cropped)
align_data_class[image_path]['aligned'][
output_filename_n] = (bb[0], bb[1], bb[2], bb[3])
if images_limit and nrof_successfully_aligned >= images_limit:
break
aligned_class_images.extend(
list(align_data_class[image_path]['aligned'].keys()))
nrof_images_skipped -= 1
if images_limit and nrof_successfully_aligned >= images_limit:
break
if os.path.isdir(output_class_dir):
cls_ = cls.name.replace(' ', '_')
if cls_ in loaded_dataset_meta:
with open(
os.path.join(output_class_dir,
dataset.META_FILENAME), 'w') as mf:
json.dump(loaded_dataset_meta[cls_], mf)
# clear not existing in input already exists aligned class images
if not self.complementary_align:
if os.path.isdir(output_class_dir):
for f in os.listdir(output_class_dir):
if f == dataset.META_FILENAME:
continue
fp = os.path.join(output_class_dir, f)
if os.path.isfile(
fp) and fp not in aligned_class_images:
os.remove(fp)
align_data[cls.name] = align_data_class
if images_limit and images_limit <= nrof_successfully_aligned:
utils.print_fun("Limit for aligned images %d is reached" %
images_limit)
break
dataset.get_meta(loaded_dataset)
# clear not existing in input already exists aligned classes (dirs)
if not self.complementary_align:
for d in os.listdir(aligned_dir):
dd = os.path.join(aligned_dir, d)
if os.path.isdir(dd) and d not in align_data:
shutil.rmtree(dd, ignore_errors=True)
with open(bounding_boxes_filename, "w") as text_file:
text_file.write(bounding_boxes_contents)
with open(align_filename, 'wb') as align_file:
pickle.dump((align_data_args, align_data), align_file, protocol=2)
utils.print_fun('Total number of images: %d' % nrof_images_total)
if nrof_images_cached > 0:
utils.print_fun('Number of cached images: %d' % nrof_images_cached)
if nrof_images_skipped > 0:
utils.print_fun('Number of skipped images: %d' %
nrof_images_skipped)
if nrof_has_meta > 0:
utils.print_fun('Number of classes with meta: %d' % nrof_has_meta)
utils.print_fun('Number of successfully aligned images: %d' %
nrof_successfully_aligned)
def _get_boxes(self, image_path, img):
serving_img = cv2.resize(img,
self.input_size,
interpolation=cv2.INTER_AREA)
if self.serving.driver_name == 'openvino':
serving_img = np.transpose(serving_img, [2, 0, 1]).reshape(
[1, 3, *self.input_size[::-1]])
else:
serving_img = serving_img.reshape([1, *self.input_size[::-1],
3]).astype(np.uint8)
raw = self.serving.predict({self.input_name: serving_img})
if self.serving.driver_name == 'edgetpu':
output = raw
score = output[2]
bboxes_raw = output[0].reshape([-1, 4])
bboxes_raw = bboxes_raw[score > self.threshold]
bounding_boxes = np.zeros_like(bboxes_raw)
# y1, x1, y2, x2 -> x1, y1, x2, y2
bounding_boxes[:, 0] = bboxes_raw[:, 1] * img.shape[1]
bounding_boxes[:, 1] = bboxes_raw[:, 0] * img.shape[0]
bounding_boxes[:, 2] = bboxes_raw[:, 3] * img.shape[1]
bounding_boxes[:, 3] = bboxes_raw[:, 2] * img.shape[0]
else:
# 7 values:
# class_id, label, confidence, x_min, y_min, x_max, y_max
# Select boxes where confidence > factor
raw = raw[self.output_name].reshape([-1, 7])
bboxes_raw = raw[raw[:, 2] > self.threshold]
bboxes_raw[:, 3] = bboxes_raw[:, 3] * img.shape[1]
bboxes_raw[:, 5] = bboxes_raw[:, 5] * img.shape[1]
bboxes_raw[:, 4] = bboxes_raw[:, 4] * img.shape[0]
bboxes_raw[:, 6] = bboxes_raw[:, 6] * img.shape[0]
bounding_boxes = np.zeros([len(bboxes_raw), 5])
bounding_boxes[:, 0:4] = bboxes_raw[:, 3:7]
bounding_boxes[:, 4] = bboxes_raw[:, 2]
# output = output.reshape(-1, 7)
# bboxes_raw = output[output[:, 2] > self.threshold]
# Extract 4 values
# bounding_boxes = bboxes_raw[:, 3:7]
# Get the biggest box: find the box with largest square:
# (y1 - y0) * (x1 - x0) - size of box.
bbs = bounding_boxes
area = (bbs[:, 3] - bbs[:, 1]) * (bbs[:, 2] - bbs[:, 0])
if len(area) < 1:
utils.print_fun('WARNING: Unable to align "%s", n_faces=%s' %
(image_path, len(area)))
return None
num = np.argmax(area)
if area[num] < self.min_face_area:
utils.print_fun(
'WARNING: Unable to align "{}", face found but too small - about {}px '
'width against required minimum of {}px. Try adjust parameter --min-face-size'
.format(image_path, int(np.sqrt(area[num])),
self.min_face_size))
return None
bounding_boxes = np.stack([bbs[num]])
return bounding_boxes
def report(tpr, fpr, accuracy, val, val_std, far):
import io
import matplotlib.pyplot as plt
plt.figure(1, figsize=(10, 10))
# plot no skill
plt.plot([0, 1], [0, 1], linestyle='--')
# plot the roc curve for the model
plt.plot(fpr, tpr, marker='.')
plt.title('ROC Curve')
plt.tight_layout(pad=1.5)
# plt.ylabel('True label')
# plt.xlabel('Predicted label')
buf = io.BytesIO()
plt.savefig(buf, format='png')
buf.seek(0)
img = cv2.imdecode(np.frombuffer(buf.getvalue(), dtype=np.uint8),
cv2.IMREAD_COLOR)
img = img[:, :, ::-1]
img = np.vstack((np.ones([150, img.shape[1], 3]) * 255, img))
mean_accuracy = np.mean(accuracy)
std_accuracy = np.std(accuracy)
utils.print_fun('Accuracy: %2.5f+-%2.5f' % (mean_accuracy, std_accuracy))
font = cv2.FONT_HERSHEY_DUPLEX
utils.print_fun('Validation rate: %2.5f+-%2.5f @ FAR=%2.5f' %
(val, val_std, far))
auc = metrics.auc(fpr, tpr)
utils.print_fun('Area Under Curve (AUC): %1.3f' % auc)
eer = brentq(lambda x: 1. - x - interpolate.interp1d(fpr, tpr)(x), 0., 1.)
utils.print_fun('Equal Error Rate (EER): %1.3f' % eer)
cv2.putText(img,
'Accuracy: %2.5f+-%2.5f' % (mean_accuracy, std_accuracy),
(50, 30),
font,
1.0,
0,
thickness=1,
lineType=cv2.LINE_AA)
cv2.putText(img,
'Validation rate: %2.5f+-%2.5f @ FAR=%2.5f' %
(val, val_std, far), (50, 70),
font,
1.0,
0,
thickness=1,
lineType=cv2.LINE_AA)
cv2.putText(img,
'Area Under Curve (AUC): %1.3f' % auc, (50, 110),
font,
1.0,
0,
thickness=1,
lineType=cv2.LINE_AA)
cv2.putText(img,
'Equal Error Rate (EER): %1.3f' % eer, (50, 150),
font,
1.0,
0,
thickness=1,
lineType=cv2.LINE_AA)
buf = cv2.imencode('.jpg', img)[1].tostring()
return '<html><img src="data:image/png;base64,{}"/></html>'.format(
base64.b64encode(buf).decode())
def main(args):
# Get the paths for the corresponding images
use_mlboard = False
mlboard = None
if client:
mlboard = client.Client()
try:
mlboard.apps.get()
except Exception:
mlboard = None
utils.print_fun('Do not use mlboard.')
else:
utils.print_fun('Use mlboard parameters logging.')
use_mlboard = True
image_size = args.image_size
driver_name = 'openvino'
if os.path.isdir(args.model) and os.path.exists(
os.path.join(args.model, 'saved_model.pb')):
driver_name = 'tensorflow'
image_size = 112
data = {
'image_size': image_size,
'driver_name': driver_name,
'model_path': args.model,
'data_dir': args.data_dir,
'batch_size': args.batch_size,
}
update_data(data, use_mlboard, mlboard)
img_paths, actual_issame = load_dataset(args.data_dir)
drv = driver.load_driver(driver_name)
serving = drv()
serving.load_model(
args.model,
inputs='input:0,phase_train:0',
outputs='embeddings:0',
device='CPU',
flexible_batch_size=True,
)
# Run forward pass to calculate embeddings
utils.print_fun('Runnning forward pass on dataset images')
# Enqueue one epoch of image paths and labels
nrof_images = len(img_paths)
data = {
'num_images': nrof_images,
'num_classes': nrof_images // 4,
}
update_data(data, use_mlboard, mlboard)
embedding_size = list(serving.outputs.values())[0][-1]
nrof_batches = int(np.ceil(float(nrof_images) / args.batch_size))
emb_array = np.zeros((nrof_images, embedding_size))
# TODO(nmakhotkin): cache embeddings by image paths (because image pairs
# are duplicated and no need to do inference on them)
for i in range(nrof_batches):
start_index = i * args.batch_size
end_index = min((i + 1) * args.batch_size, nrof_images)
paths_batch = img_paths[start_index:end_index]
probe_imgs = dataset.load_data(paths_batch,
image_size,
normalization=args.normalization)
emb = _predict(serving, probe_imgs)
emb_array[start_index:end_index, :] = emb
if i % 5 == 4:
utils.print_fun('{}/{}'.format(i + 1, nrof_batches))
sys.stdout.flush()
utils.print_fun('')
embeddings = emb_array
tpr, fpr, accuracy, val, val_std, far = helpers.evaluate(
embeddings,
actual_issame,
nrof_folds=args.lfw_nrof_folds,
distance_metric=args.distance_metric,
subtract_mean=args.subtract_mean)
rpt = report(tpr, fpr, accuracy, val, val_std, far)
with open('report.html', 'w') as f:
f.write(rpt)
update_data({'#documents.report.html': rpt}, use_mlboard, mlboard)
def __init__(self, bg_remove_path):
utils.print_fun('Load BG_REMOVE model')
drv = sdrv.load_driver('tensorflow')
self.drv = drv()
self.drv.load_model(bg_remove_path)
from app.tools import utils
try:
from mlboardclient.api import client
except ImportError:
client = None
mlboard = None
mlboard_logging = False
if client:
mlboard = client.Client()
mlboard_logging = True
try:
mlboard.apps.get()
except Exception:
mlboard_logging = False
utils.print_fun('Do not use mlboard parameters logging.')
else:
utils.print_fun('Using mlboard parameters logging.')
def update_task_info(data):
if mlboard and mlboard_logging:
mlboard.update_task_info(data)
def catalog_ref(name, ctype, version):
return '#/{}/catalog/{}/{}/versions/{}'. \
format(os.environ.get('WORKSPACE_NAME'), ctype, name, version)
