def main():
ml = client.Client()
current_task_name = os.environ.get('TASK_NAME')
LOG.info("Current task name = %s" % current_task_name)
app = ml.apps.get()
for task in app.tasks:
if task.name == current_task_name:
continue
LOG.info("Start task %s..." % task.name)
started = task.start()
LOG.info("Run & wait [name=%s, build=%s, status=%s]" %
(started.name, started.build, started.status))
completed = started.wait()
if completed.status != SUCCEEDED:
LOG.warning("Task %s-%s completed with status %s." %
(completed.name, completed.build, completed.status))
LOG.warning("Workflow is completed with status ERROR")
sys.exit(1)
LOG.info("Task %s-%s completed with status %s." %
(completed.name, completed.build, completed.status))
LOG.info("Workflow is completed with status SUCCESS")
def main():
m = client.Client()
args = parse_args()
data_dir = path.join(args.train_dir, 'mnist_sample')
data_path = untar_data(URLs.MNIST_SAMPLE,
fname=tempfile.mktemp(),
dest=data_dir)
print('Using path %s' % data_path)
m.update_task_info({'data_path': str(data_path)})
data = ImageDataBunch.from_folder(data_path,
ds_tfms=(rand_pad(2, 28), []),
bs=64)
data.normalize(imagenet_stats)
learn = create_cnn(data, models.resnet18, metrics=accuracy)
learn.fit_one_cycle(1, 0.01)
print('Accuracy %s' % str(accuracy(*learn.get_preds())))
m.update_task_info({'accuracy': str(accuracy(*learn.get_preds()))})
model_location = path.join(args.train_dir, "model")
model_location = learn.save(model_location, return_path=True)
print('Model saved to %s' % model_location)
m.update_task_info({'model_location': str(model_location)})
print('Network structure:')
learn.model.eval()
def main(_):
img_dir = '/tmp/mnist'
logging.info("Extracting images to %s ...", img_dir)
mnist = input_data.read_data_sets(FLAGS.data_dir,
one_hot=True,
source_url=FLAGS.source_url)
os.mkdir(img_dir)
for i in range(mnist.test.num_examples):
batch = mnist.test.next_batch(1)
v = np.reshape(batch[0][0], (28, 28)) * 255
l = batch[1][0]
l = l.astype(int)
n = 0
for k in l:
if k == 1:
break
n += 1
l = ('%s/%d-%d.png') % (img_dir, i, n)
scipy.misc.imsave(l, v)
logging.info("Pushing dataset to %s:%s ...", FLAGS.catalog_name,
FLAGS.version)
kl = client.Client()
kl.datasets.push(os.environ.get('WORKSPACE_NAME'),
FLAGS.catalog_name,
FLAGS.version,
img_dir,
create=True)
logging.info("Push success")
def main():
ml = client.Client()
current_task_name = os.environ.get('TASK_NAME')
LOG.info("Current task name = %s" % current_task_name)
current_project = os.environ['PROJECT_NAME']
current_workspace = os.environ['WORKSPACE_ID']
LOG.info("Current project = %s" % current_project)
LOG.info("Current workspace = %s" % current_workspace)
current_app_id = current_workspace + '-' + current_project
app = ml.apps.get(current_app_id)
for task in app.tasks:
if task.name == current_task_name:
continue
LOG.info("Start task %s..." % task.name)
started = task.start()
LOG.info("Run & wait [name=%s, build=%s, status=%s]" %
(started.name, started.build, started.status))
completed = started.wait()
if completed.status != SUCCEEDED:
LOG.warning("Task %s-%s completed with status %s." %
(completed.name, completed.build, completed.status))
LOG.warning("Workflow is completed with status ERROR")
sys.exit(1)
LOG.info("Task %s-%s completed with status %s." %
(completed.name, completed.build, completed.status))
LOG.info("Workflow is completed with status SUCCESS")
def export(checkpoint_dir, task_name, build_id, catalog_name):
featureX = tf.contrib.layers.real_valued_column('x')
featureY = tf.contrib.layers.real_valued_column('y')
model = tf.contrib.learn.SVM(example_id_column='i',
feature_columns=[featureX, featureY],
model_dir=checkpoint_dir)
feature_spec = {
'x': tf.FixedLenFeature(dtype=tf.float32, shape=[1]),
'y': tf.FixedLenFeature(dtype=tf.float32, shape=[1])
}
serving_fn = tf.contrib.learn.utils.input_fn_utils.build_parsing_serving_input_fn(
feature_spec)
export_path = model.export_savedmodel(export_dir_base=checkpoint_dir,
serving_input_fn=serving_fn)
export_path = export_path.decode("utf-8")
logging.info("\nModel Path: %s", export_path)
client.update_task_info({'model_path': export_path},
task_name=task_name,
build_id=build_id)
client.update_task_info({
'model_path': export_path,
'checkpoint_path': checkpoint_dir
})
if catalog_name is not None:
ml = client.Client()
ml.model_upload(catalog_name, '1.0.' + build_id, export_path)
def __init__(self,
checkpoint_dir,
tensors=None,
submit_summary=True,
max_number_images=1,
every_steps=None,
every_n_secs=60):
if tensors is None:
tensors = {}
if every_steps is not None:
every_n_secs = None
self._timer = tf.train.SecondOrStepTimer(every_steps=every_steps,
every_secs=every_n_secs)
if submit_summary:
self._rpt = Report(checkpoint_dir,
max_number_images=max_number_images)
else:
self._rpt = None
try:
from mlboardclient.api import client
except ImportError:
tf.logging.warning("Can't find mlboardclient.api")
client = None
mlboard = None
if client:
mlboard = client.Client()
try:
mlboard.apps.get()
except Exception:
tf.logging.warning("Can't init mlboard env")
mlboard = None
self._mlboard = mlboard
self._tensors = tensors
self._most_recent_step = 0
def export(checkpoint_dir, params):
m = client.Client()
base_id = '0'
if os.environ.get('BASE_TASK_BUILD_ID', '') != '':
app = m.apps.get()
base_id = os.environ['BASE_TASK_BUILD_ID']
task_name = os.environ['BASE_TASK_NAME']
task = app.get_task(task_name, base_id)
checkpoint_dir = task.exec_info['checkpoint_path']
params['num_chans'] = task.exec_info['num-chans']
params['num_pools'] = task.exec_info['num-pools']
params['resolution'] = task.exec_info['resolution']
params['checkpoint'] = checkpoint_dir
conf = tf.estimator.RunConfig(
model_dir=checkpoint_dir,
)
params['batch_size'] = 1
features_def = [int(i) for i in task.exec_info.get('features','3').split(',')]
logging.info('Features Def: {}'.format(features_def))
params['features'] = features_def
feature_placeholders = {
'image': tf.placeholder(tf.float32, [1, None, None, sum(features_def)], name='image'),
}
receiver = tf.estimator.export.build_raw_serving_input_receiver_fn(feature_placeholders)
net = FastBGNet(
params=params,
model_dir=checkpoint_dir,
config=conf,
)
models = os.path.join(checkpoint_dir, 'models')
build_id = os.environ['BUILD_ID']
export_dir = os.path.join(models, build_id)
os.makedirs(export_dir, exist_ok=True)
export_path = net.export_savedmodel(
export_dir,
receiver,
)
export_path = export_path.decode("utf-8")
base = os.path.basename(export_path)
driver_data = {'driver': 'tensorflow', 'path': base}
with open(os.path.join(export_dir, '_model_config.yaml'), 'w') as f:
yaml.dump(driver_data, f)
params['num_chans'] = task.exec_info['num-chans']
params['num_pools'] = task.exec_info['num-pools']
params['resolution'] = task.exec_info['resolution']
version = f'1.{base_id}.{build_id}'
model_name = 'person-mask'
m.model_upload(model_name, version, export_dir, spec=serving_spec())
client.update_task_info({'model_path': export_path, 'num-chans': params['num_chans'],
'features':','.join([str(i) for i in features_def]),
'num-pools': params['num_pools'], 'resolution': params['resolution'],
'model_reference': catalog_ref(model_name, 'mlmodel', version)})
def main(_):
if FLAGS.mode == 'export':
log_dir = os.environ.get('TRAINING_DIR') + '/' + FLAGS.build
client.update_task_info({'checkpoint_path': log_dir})
export_path = os.path.join(log_dir, str(FLAGS.model_version))
export(export_path, log_dir)
client.update_task_info({'model_path': export_path}, task_name='train', build_id=FLAGS.build)
if FLAGS.catalog_name is not None:
ml = client.Client()
ml.model_upload(FLAGS.catalog_name, '1.0.' + FLAGS.build, export_path)
else:
train()
def push_model(target, dirame):
if os.environ.get('PROJECT_ID', None):
from mlboardclient.api import client
timestamp = datetime.datetime.now().strftime('%s')
if target is not None:
version = '1.0.0-openvino-{}-{}'.format(target, timestamp)
else:
version = '1.0.0-openvino-{}'.format(timestamp)
mlboard = client.Client()
mlboard.model_upload('facenet', version, dirame)
submit({'model': catalog_ref('facenet', 'mlmodel', version)})
logging.info(
"New model uploaded as 'facenet', version '{}'.".format(version))
def push(name, dirame):
if os.environ.get('PROJECT_ID', None):
from mlboardclient.api import client
timestamp = datetime.datetime.now().strftime('%s')
if name is not None:
version = '1.0.0-{}-{}'.format(name, timestamp)
else:
version = '1.0.0-{}'.format(timestamp)
mlboard = client.Client()
mlboard.model_upload('openvino-facenet', version, dirame)
submit({'model': '{}:{}'.format('openvino-facenet', version)})
logging.info(
"New model uploaded as 'openvino-facenet', version '%s'." %
(version))
def get():
if client is None:
return None
global mlboard, mlboard_tried
if not mlboard_tried:
mlboard_tried = True
mlboard = client.Client()
try:
mlboard.apps.get()
except Exception:
mlboard = None
logging.info('Do not use mlboard.')
else:
logging.info('Use mlboard parameters logging.')
return mlboard
def main(_):
data_dir = "/tmp/mnist"
logging.info("Uploading dataset to %s ...", data_dir)
input_data.read_data_sets(data_dir,
one_hot=True,
source_url=FLAGS.source_url)
logging.info("Upload success")
logging.info("Pushing dataset to %s:%s ...", FLAGS.catalog_name,
FLAGS.version)
kl = client.Client()
kl.datasets.push(os.environ.get('WORKSPACE_NAME'),
FLAGS.catalog_name,
FLAGS.version,
data_dir,
create=True)
logging.info("Push success")
def after_export(training_dir, train_build_id, model_name, model_version):
m = client.Client()
m.model_upload(
model_name,
model_version,
'{}/model/{}/saved_model'.format(training_dir, train_build_id),
)
m.update_task_info({
'model':
'#/%s/catalog/mlmodel/%s/versions/%s' % (
os.environ['WORKSPACE_NAME'],
model_name,
model_version,
),
})
def main():
parser = get_parser()
args = parser.parse_args()
if args.debug:
logging.root.setLevel('DEBUG')
m = client.Client()
app = m.apps.get()
task = app.task('train')
if args.data_version!='':
task.config['datasetRevisions']=[{'volumeName': 'data', 'revision': args.data_version}]
task.resource('worker')['command'] = 'python svm.py --steps=1000 --checkpoint_dir=$TRAINING_DIR/$BUILD_ID'
spec = (optimizator.ParamSpecBuilder().resource('worker')
.param('l2_regularization')
.bounds(1,10)
.param('l1_regularization')
.bounds(1,10)
.build())
LOG.info('Run with param spec = %s', spec)
result = task.optimize(
'accuracy',
spec,
init_steps=args.init_steps,
iterations=args.iterations,
method=args.method,
max_parallel=args.parallel,
direction='maximize'
)
best = result['best']
LOG.info('Found best build %s:%s: %.2f', best.name,best.build,best.exec_info['accuracy'])
client.update_task_info({'checkpoint_path':best.exec_info['checkpoint_path'],
'accuracy':best.exec_info['accuracy'],'build':best.build})
LOG.info('Exporting model %s...',best.build)
export = app.task('export')
export.resource('run')['command'] = 'python svm.py --export_model'
export.resource('run')['args']= {
'catalog_name': 'my_svm_model',
'task_name': best.name,
'build_id': best.build,
'checkpoint_dir': best.exec_info['checkpoint_path']
}
export.start()
export.wait()
client.update_task_info({'model_path':export.exec_info['model_path']})
def push_dataset(target, dirname):
if os.environ.get('PROJECT_ID', None):
from mlboardclient.api import client
timestamp = datetime.datetime.now().strftime('%s')
if target is not None:
version = '1.0.0-openvino-{}-{}'.format(target, timestamp)
else:
version = '1.0.0-openvino-{}'.format(timestamp)
mlboard = client.Client()
mlboard.datasets.push(os.environ.get('WORKSPACE_NAME'),
'facenet-pretrained',
version,
dirname,
create=True)
submit(
{'model': catalog_ref('facenet-pretrained', 'dataset', version)})
logging.info("New model uploaded as 'facenet-pretrained', "
"version '{}'.".format(version))
def main():
build_config()
parser = ArgumentParser()
parser.add_argument('--research_dir')
parser.add_argument('--training_dir')
parser.add_argument('--model_name', default="object-detection")
parser.add_argument('--model_version', default="1.0.0")
parser.add_argument('--train_build_id')
parser.add_argument('--train_checkpoint')
args, _ = parser.parse_known_args()
targs = sys.argv[:]
targs[0] = args.research_dir + '/object_detection/export_inference_graph.py'
targs.insert(0, sys.executable or 'python')
targs.append("--pipeline_config_path")
targs.append("faster_rcnn.config")
targs.append("--trained_checkpoint_prefix")
targs.append("%s/%s/model.ckpt-%s" % (args.training_dir, args.train_build_id, args.train_checkpoint))
targs.append("--output_directory")
targs.append("%s/model/%s" % (args.training_dir, args.train_build_id))
targs.append("--input_type")
targs.append("encoded_image_string_tensor")
print("Execute: ", targs)
call(targs)
m = client.Client()
m.model_upload(
args.model_name,
args.model_version,
'%s/model/%s/saved_model' % (args.training_dir, args.train_build_id),
)
m.update_task_info({
'model': '#/%s/catalog/mlmodel/%s/versions/%s' % (
os.environ['WORKSPACE_NAME'],
args.model_name,
args.model_version,
),
})
def main():
parser = get_parser()
args = parser.parse_args()
if args.debug:
logging.root.setLevel('DEBUG')
m = client.Client()
if args.task != '':
app = m.apps.get()
serving = app.servings[0]
srv = serving.start(args.task, args.build)
LOG.info("Start serving")
name = os.environ.get('PROJECT_NAME') + '-' + srv.config[
'name'] + '-' + args.task + '-' + args.build
LOG.info("Waiting serving ready...")
status = 0
for i in range(6):
time.sleep(10)
resp = m.servings.call(
name,
'svm', {"features": [{
"x": {
"Float": 0
},
"y": {
"Float": 0
}
}]},
port='9000')
status = resp.status_code
if status == 200:
break
if status != 200:
srv.stop()
raise RuntimeError('Failed start serving')
res = validate(m, name)
srv.stop()
if res != '':
raise RuntimeError(res)
else:
res = validate(m, args.model)
if res != '':
raise RuntimeError(res)
def main():
parser = get_parser()
args = parser.parse_args()
if args.debug:
logging.root.setLevel('DEBUG')
ml = client.Client()
app = ml.apps.get()
def _task(i, task):
step_count = 100 + i * 300
model = 'svm-s_%d-l1_%d-l2_%d' % (step_count, 1, 10)
task.comment = model
task.resource('worker')['args'] = {
'l2_regularization':
10,
'l1_regularization':
1,
'steps':
step_count,
'checkpoint_dir':
'%s/%s-%s' %
(os.environ.get('TRAINING_DIR'), os.environ.get('BUILD_ID'), model)
}
def jobs(num):
for i in range(num):
yield partial(_task, i)
task = app.task('train')
results = task.parallel_run(2, jobs(args.runs))
best_accuracy = 0
best = None
for r in results:
if r.exec_info['accuracy'] > best_accuracy:
best_accuracy = r.exec_info['accuracy']
best = r
LOG.info('BEST %s with accuracy=%.2f', best.exec_info['model_path'],
best_accuracy)
def main():
build_config()
with open('faster_rcnn.config', 'r') as cf:
data = cf.read()
config_html = '<html><head></head><body><pre style="word-wrap: break-word; white-space: pre-wrap;">{}</pre></body></html>'.format(
data)
client.Client().update_task_info({'#documents.config.html': config_html})
parser = ArgumentParser()
parser.add_argument('--training_dir')
parser.add_argument('--research_dir')
parser.add_argument('--model_name', default="object-detection")
parser.add_argument('--model_version', default="1.0.0")
parser.add_argument('--build_id')
parser.add_argument('--num_steps')
args, _ = parser.parse_known_args()
export_subprocess(args.research_dir, args.training_dir, args.build_id,
args.num_steps, args.model_name, args.model_version)
def main():
build_config()
parser = ArgumentParser()
parser.add_argument('--training_dir')
parser.add_argument('--research_dir')
parser.add_argument('--build_id')
parser.add_argument('--num_steps', default=1000)
parser.add_argument('--only_train', default='False')
args, _ = parser.parse_known_args()
targs = sys.argv[:]
targs[0] = args.research_dir + '/object_detection/model_main.py'
targs.insert(0, sys.executable or 'python')
targs.append("--pipeline_config_path")
targs.append("faster_rcnn.config")
targs.append("--model_dir")
targs.append("%s/%s" % (args.training_dir, args.build_id))
call(targs)
client.Client().update_task_info({'train_build_id': args.build_id})
def main():
parser = get_parser()
args = parser.parse_args()
if args.debug:
logging.root.setLevel('DEBUG')
m = client.Client()
app = m.apps.get()
task = app.task('train')
task.resource(
'worker'
)['command'] = 'python mnist.py --training_iteration=1000 --model_version 0'
spec = (optimizator.ParamSpecBuilder().resource('worker').param(
'fully_neurons').int().bounds(1,
3).param('drop_out').bounds(0.2,
0.7).build())
LOG.info('Run with param spec = %s', spec)
result = task.optimize('test_accuracy',
spec,
init_steps=args.init_steps,
iterations=args.iterations,
method=args.method,
max_parallel=args.parallel,
direction='maximize')
best = result['best']
LOG.info('Found best build %s:%s: %.2f', best.name, best.build,
best.exec_info['test_accuracy'])
LOG.info('Exporting model to catalog mnist/%s', best.build)
export = app.task('export')
export.resource('run')['command'] = 'python mnist.py'
export.resource('run')['args'] = {
'mode': 'export',
'catalog_name': 'mnist',
'build': best.build,
'model_version': 1
}
export.start()
export.wait()
def main():
parser = get_parser()
args = parser.parse_args()
dataset = '/tmp/data'
if not os.path.exists(dataset):
os.makedirs(dataset)
train = pd.DataFrame(random.randint(low=0, high=100, size=(1000, 2)),
columns=['x', 'y'])
train['label'] = train.apply(lambda v: 0 if v['x'] > v['y'] +
(5 - random.random_sample() * 10) else 1,
axis=1)
test = pd.DataFrame(random.randint(low=0, high=100, size=(100, 2)),
columns=['x', 'y'])
test['label'] = test.apply(lambda v: 0 if v['x'] > v['y'] else 1, axis=1)
train.to_csv(dataset + '/train.csv')
test.to_csv(dataset + '/test.csv')
kl = client.Client()
kl.datasets.push(os.environ.get('WORKSPACE_NAME'),
args.dataset,
args.version,
dataset,
create=True)
client.update_task_info(
{'dataset': '%s:%s' % (args.dataset, args.version)})
def main(args):
use_mlboard = False
mlboard = None
if client:
mlboard = client.Client()
try:
mlboard.apps.get()
except Exception:
mlboard = None
print('Do not use mlboard.')
else:
print('Use mlboard parameters logging.')
use_mlboard = True
with tf.Graph().as_default():
with tf.Session() as sess:
np.random.seed(seed=args.seed)
if args.use_split_dataset:
dataset_tmp = facenet.get_dataset(args.data_dir)
train_set, test_set = split_dataset(
dataset_tmp, args.min_nrof_images_per_class,
args.nrof_train_images_per_class)
if args.mode == 'TRAIN':
dataset = train_set
elif args.mode == 'CLASSIFY':
dataset = test_set
else:
dataset = facenet.get_dataset(args.data_dir)
update_data({'mode': args.mode}, use_mlboard, mlboard)
# Check that there are at least one training image per class
for cls in dataset:
assert len(
cls.image_paths
) > 0, 'There must be at least one image for each class in the dataset'
paths, labels = facenet.get_image_paths_and_labels(dataset)
print('Number of classes: %d' % len(dataset))
print('Number of images: %d' % len(paths))
data = {
'num_classes': len(dataset),
'num_images': len(paths),
'model_path': args.model,
'image_size': args.image_size,
'data_dir': args.data_dir,
'batch_size': args.batch_size,
}
update_data(data, use_mlboard, mlboard)
# Load the model
print('Loading feature extraction model')
facenet.load_model(args.model)
# Get input and output tensors
images_placeholder = tf.get_default_graph().get_tensor_by_name(
"input:0")
embeddings = tf.get_default_graph().get_tensor_by_name(
"embeddings:0")
phase_train_placeholder = tf.get_default_graph(
).get_tensor_by_name("phase_train:0")
embedding_size = embeddings.get_shape()[1]
# Run forward pass to calculate embeddings
print('Calculating features for images')
nrof_images = len(paths)
nrof_batches_per_epoch = int(
math.ceil(1.0 * nrof_images / args.batch_size))
emb_array = np.zeros((nrof_images, embedding_size))
for i in range(nrof_batches_per_epoch):
start_index = i * args.batch_size
end_index = min((i + 1) * args.batch_size, nrof_images)
paths_batch = paths[start_index:end_index]
for j in range(end_index - start_index):
print('Batch {} <-> {}'.format(paths_batch[j],
labels[start_index + j]))
images = facenet.load_data(paths_batch, False, False,
args.image_size)
feed_dict = {
images_placeholder: images,
phase_train_placeholder: False
}
emb_array[start_index:end_index, :] = sess.run(
embeddings, feed_dict=feed_dict)
classifier_filename_exp = os.path.expanduser(
args.classifier_filename)
if args.mode == 'TRAIN':
# Train classifier
print('Training classifier')
model = svm.SVC(kernel='linear', probability=True)
model.fit(emb_array, labels)
# Create a list of class names
class_names = [cls.name.replace('_', ' ') for cls in dataset]
print('Classes:')
print(class_names)
# Saving classifier model
with open(classifier_filename_exp, 'wb') as outfile:
pickle.dump((model, class_names), outfile, protocol=2)
print('Saved classifier model to file "%s"' %
classifier_filename_exp)
elif args.mode == 'CLASSIFY':
# Classify images
print('Testing classifier')
with open(classifier_filename_exp, 'rb') as infile:
(model, class_names) = pickle.load(infile)
print('Loaded classifier model from file "%s"' %
classifier_filename_exp)
predictions = model.predict_proba(emb_array)
best_class_indices = np.argmax(predictions, axis=1)
best_class_probabilities = predictions[
np.arange(len(best_class_indices)), best_class_indices]
for i in range(len(best_class_indices)):
print('%4d %s: %.3f' %
(i, class_names[best_class_indices[i]],
best_class_probabilities[i]))
accuracy = np.mean(np.equal(best_class_indices, labels))
update_data({'accuracy': accuracy}, use_mlboard, mlboard)
print('Accuracy: %.3f' % accuracy)
if args.upload_model and accuracy >= args.upload_threshold:
timestamp = datetime.datetime.now().strftime('%s')
upload_model(use_mlboard, mlboard, classifier_filename_exp,
'facenet-classifier', '1.0.0-%s' % timestamp)
##
# Runs train -> eval -> export.
##
import argparse
import logging
import re, sys
from mlboardclient.api import client
SUCCEEDED = 'Succeeded'
logging.basicConfig(level=logging.INFO)
LOG = logging.getLogger(__name__)
mlboard = client.Client()
run_tasks = [
'train',
'export',
]
def override_task_arguments(task, params):
for k, v in params.items():
pattern = re.compile('--{}[ =]([^\s]+|[\'"].*?[\'"])'.format(k))
resource = task.config['resources'][0]
task_cmd = resource['command']
replacement = '--{} {}'.format(k, v)
if pattern.findall(task_cmd):
# Replace
resource['command'] = pattern.sub(replacement, task_cmd)
else:
# Add
def main(args):
algorithms = ["kNN", "SVM"]
use_mlboard = False
mlboard = None
if client:
mlboard = client.Client()
try:
mlboard.apps.get()
except Exception:
mlboard = None
print_fun('Do not use mlboard.')
else:
print_fun('Use mlboard parameters logging.')
use_mlboard = True
if args.use_split_dataset:
dataset_tmp = facenet.get_dataset(args.data_dir)
train_set, test_set = split_dataset(dataset_tmp, args.min_nrof_images_per_class,
args.nrof_train_images_per_class)
if args.mode == 'TRAIN':
dataset = train_set
elif args.mode == 'CLASSIFY':
dataset = test_set
else:
dataset = facenet.get_dataset(args.data_dir)
update_data({'mode': args.mode}, use_mlboard, mlboard)
# Check that there are at least one training image per class
for cls in dataset:
if len(cls.image_paths) == 0:
print_fun('WARNING: %s: There are no aligned images in this class.' % cls)
paths, labels = facenet.get_image_paths_and_labels(dataset)
print_fun('Number of classes: %d' % len(dataset))
print_fun('Number of images: %d' % len(paths))
data = {
'num_classes': len(dataset),
'num_images': len(paths),
'model_path': args.model,
'image_size': args.image_size,
'data_dir': args.data_dir,
'batch_size': args.batch_size,
}
update_data(data, use_mlboard, mlboard)
# Load the model
print_fun('Loading feature extraction model')
# Load and instantinate driver
drv = driver.load_driver(args.driver)
serving = drv()
serving.load_model(
args.model,
inputs='input:0,phase_train:0',
outputs='embeddings:0',
device=args.device,
flexible_batch_size=True,
)
# Run forward pass to calculate embeddings
print_fun('Calculating features for images')
noise_count = max(0, args.noise_count) if args.noise else 0
emb_args = {
'model': args.model,
'use_split_dataset': args.use_split_dataset,
'noise': noise_count > 0,
'noise_count': noise_count,
'flip': args.flip,
'image_size': args.image_size,
'min_nrof_images_per_class': args.min_nrof_images_per_class,
'nrof_train_images_per_class': args.nrof_train_images_per_class,
}
stored_embeddings = {}
if args.mode == 'TRAIN':
embeddings_filename = os.path.join(
args.data_dir,
"embeddings-%s.pkl" % hashlib.md5(json.dumps(emb_args, sort_keys=True).encode()).hexdigest(),
)
if os.path.isfile(embeddings_filename):
print_fun("Found stored embeddings data, loading...")
with open(embeddings_filename, 'rb') as embeddings_file:
stored_embeddings = pickle.load(embeddings_file)
total_time = 0.
nrof_images = len(paths)
nrof_batches_per_epoch = int(math.ceil(1.0 * nrof_images / args.batch_size))
epp = embeddings_per_path(noise_count, args.flip)
embeddings_size = nrof_images * epp
emb_array = np.zeros((embeddings_size, 512))
fit_labels = []
emb_index = 0
for i in range(nrof_batches_per_epoch):
start_index = i * args.batch_size
end_index = min((i + 1) * args.batch_size, nrof_images)
paths_batch = paths[start_index:end_index]
labels_batch = labels[start_index:end_index]
# has_not_stored_embeddings = False
paths_batch_load, labels_batch_load = [], []
for j in range(end_index - start_index):
# print_fun(os.path.split(paths_batch[j]))
cls_name = dataset[labels_batch[j]].name
cached = True
if cls_name not in stored_embeddings or paths_batch[j] not in stored_embeddings[cls_name]:
# has_not_stored_embeddings = True
cached = False
paths_batch_load.append(paths_batch[j])
labels_batch_load.append(labels_batch[j])
else:
embeddings = stored_embeddings[cls_name][paths_batch[j]]
emb_array[emb_index:emb_index + len(embeddings), :] = stored_embeddings[cls_name][paths_batch[j]]
fit_labels.extend([labels_batch[j]] * len(embeddings))
emb_index += len(embeddings)
print_fun('Batch {} <-> {} {} {}'.format(
paths_batch[j], labels_batch[j], cls_name, "cached" if cached else "",
))
if len(paths_batch_load) == 0:
continue
images = load_data(paths_batch_load, labels_batch_load, args.image_size, noise_count, args.flip)
if serving.driver_name == 'tensorflow':
feed_dict = {'input:0': images, 'phase_train:0': False}
elif serving.driver_name == 'openvino':
input_name = list(serving.inputs.keys())[0]
# Transpose image for channel first format
images = images.transpose([0, 3, 1, 2])
feed_dict = {input_name: images}
else:
raise RuntimeError('Driver %s currently not supported' % serving.driver_name)
t = time.time()
outputs = serving.predict(feed_dict)
total_time += time.time() - t
emb_outputs = list(outputs.values())[0]
if args.mode == "TRAIN":
for n, e in enumerate(emb_outputs):
cls_name = dataset[labels_batch_load[n]].name
if cls_name not in stored_embeddings:
stored_embeddings[cls_name] = {}
path = paths_batch_load[n]
if path not in stored_embeddings[cls_name]:
stored_embeddings[cls_name][path] = []
stored_embeddings[cls_name][path].append(e)
emb_array[emb_index:emb_index + len(images), :] = emb_outputs
fit_labels.extend(labels_batch_load)
emb_index += len(images)
# average_time = total_time / embeddings_size * 1000
# print_fun('Average time: %.3fms' % average_time)
classifiers_path = os.path.expanduser(args.classifiers_path)
if args.mode == 'TRAIN':
# Save embeddings
with open(embeddings_filename, 'wb') as embeddings_file:
pickle.dump(stored_embeddings, embeddings_file, protocol=2)
# Clear (or create) classifiers directory
try:
shutil.rmtree(classifiers_path, ignore_errors=True)
except:
pass
os.makedirs(classifiers_path)
# Create a list of class names
dataset_class_names = [cls.name for cls in dataset]
class_names = [cls.replace('_', ' ') for cls in dataset_class_names]
print_fun('Classes:')
print_fun(class_names)
class_stats = [{} for _ in range(len(dataset_class_names))]
for cls in stored_embeddings:
class_stats[dataset_class_names.index(cls)] = {
'images': len(stored_embeddings[cls]),
'embeddings': sum(len(e) for e in stored_embeddings[cls].values()),
}
# Train classifiers
for algorithm in algorithms:
if args.only_algorithm is not None and algorithm != args.only_algorithm:
continue
print_fun('Classifier algorithm %s' % algorithm)
# update_data({'classifier_algorithm': args.algorithm}, use_mlboard, mlboard)
if algorithm == 'SVM':
model = svm.SVC(kernel='linear', probability=True)
elif algorithm == 'kNN':
# n_neighbors = int(round(np.sqrt(len(emb_array))))
model = neighbors.KNeighborsClassifier(n_neighbors=args.knn_neighbors, weights='distance')
else:
raise RuntimeError("Classifier algorithm %s not supported" % algorithm)
model.fit(emb_array, fit_labels)
# Saving classifier model
classifier_filename = get_classifier_path(classifiers_path, algorithm)
with open(classifier_filename, 'wb') as outfile:
pickle.dump((model, class_names, class_stats), outfile, protocol=2)
print_fun('Saved classifier model to file "%s"' % classifier_filename)
# update_data({'average_time_%s': '%.3fms' % average_time}, use_mlboard, mlboard)
elif args.mode == 'CLASSIFY':
summary_accuracy = 1
# Classify images
for algorithm in algorithms:
print_fun('Testing classifier %s' % algorithm)
classifier_filename = get_classifier_path(classifiers_path, algorithm)
with open(classifier_filename, 'rb') as infile:
(model, class_names, class_stats) = pickle.load(infile)
print_fun('Loaded classifier model from file "%s"' % classifier_filename)
predictions = model.predict_proba(emb_array)
best_class_indices = np.argmax(predictions, axis=1)
if isinstance(model, neighbors.KNeighborsClassifier):
param_name = 'distance'
# clf_name = "knn"
(closest_distances, _) = model.kneighbors(emb_array)
eval_values = closest_distances[:, 0]
elif isinstance(model, svm.SVC):
param_name = 'probability'
# clf_name = "svm"
eval_values = predictions[np.arange(len(best_class_indices)), best_class_indices]
else:
raise RuntimeError("Unsupported classifier type: %s" % type(model))
for i in range(len(best_class_indices)):
predicted = best_class_indices[i]
if predicted == labels[i]:
print_fun('%4d %s: %s %.3f' % (
i, class_names[predicted], param_name, eval_values[i],
))
else:
print_fun('%4d %s: %s %.3f, WRONG! Should be %s.' % (
i, class_names[predicted], param_name, eval_values[i], class_names[labels[i]]),
)
accuracy = np.mean(np.equal(best_class_indices, labels))
summary_accuracy = min(summary_accuracy, accuracy)
rpt = confusion(labels, best_class_indices, class_names,
use_mlboard and not args.skip_draw_confusion_matrix)
data = {
'accuracy': accuracy,
# 'average_time': '%.3fms' % average_time
}
if not args.skip_draw_confusion_matrix:
data['#documents.confusion_matrix.html'] = rpt
update_data(data, use_mlboard, mlboard)
print_fun('Accuracy for %s: %.3f' % (algorithm, accuracy))
if args.upload_model and summary_accuracy >= args.upload_threshold:
timestamp = datetime.datetime.now().strftime('%s')
model_name = 'facenet-classifier'
if args.device == 'MYRIAD':
model_name = model_name + "-movidius"
version = '1.0.0-%s-%s' % (args.driver, timestamp)
print_fun('Uploading model as %s:%s' % (model_name, version))
upload_model(
use_mlboard,
mlboard,
classifiers_path,
model_name,
version
)
def main(args):
# Get the paths for the corresponding images
use_mlboard = False
mlboard = None
if client:
mlboard = client.Client()
try:
mlboard.apps.get()
except Exception:
mlboard = None
utils.print_fun('Do not use mlboard.')
else:
utils.print_fun('Use mlboard parameters logging.')
use_mlboard = True
image_size = args.image_size
driver_name = 'openvino'
if os.path.isdir(args.model) and os.path.exists(
os.path.join(args.model, 'saved_model.pb')):
driver_name = 'tensorflow'
image_size = 112
data = {
'image_size': image_size,
'driver_name': driver_name,
'model_path': args.model,
'data_dir': args.data_dir,
'batch_size': args.batch_size,
}
update_data(data, use_mlboard, mlboard)
img_paths, actual_issame = load_dataset(args.data_dir)
drv = driver.load_driver(driver_name)
serving = drv()
serving.load_model(
args.model,
inputs='input:0,phase_train:0',
outputs='embeddings:0',
device='CPU',
flexible_batch_size=True,
)
# Run forward pass to calculate embeddings
utils.print_fun('Runnning forward pass on dataset images')
# Enqueue one epoch of image paths and labels
nrof_images = len(img_paths)
data = {
'num_images': nrof_images,
'num_classes': nrof_images // 4,
}
update_data(data, use_mlboard, mlboard)
embedding_size = list(serving.outputs.values())[0][-1]
nrof_batches = int(np.ceil(float(nrof_images) / args.batch_size))
emb_array = np.zeros((nrof_images, embedding_size))
# TODO(nmakhotkin): cache embeddings by image paths (because image pairs
# are duplicated and no need to do inference on them)
for i in range(nrof_batches):
start_index = i * args.batch_size
end_index = min((i + 1) * args.batch_size, nrof_images)
paths_batch = img_paths[start_index:end_index]
probe_imgs = dataset.load_data(paths_batch,
image_size,
normalization=args.normalization)
emb = _predict(serving, probe_imgs)
emb_array[start_index:end_index, :] = emb
if i % 5 == 4:
utils.print_fun('{}/{}'.format(i + 1, nrof_batches))
sys.stdout.flush()
utils.print_fun('')
embeddings = emb_array
tpr, fpr, accuracy, val, val_std, far = helpers.evaluate(
embeddings,
actual_issame,
nrof_folds=args.lfw_nrof_folds,
distance_metric=args.distance_metric,
subtract_mean=args.subtract_mean)
rpt = report(tpr, fpr, accuracy, val, val_std, far)
with open('report.html', 'w') as f:
f.write(rpt)
update_data({'#documents.report.html': rpt}, use_mlboard, mlboard)
import json
from mlboardclient.api import client
m = client.Client('http://localhost:8082/api/v2')
app = m.apps.get('11-tfexample')
task = app.task('model')
task.resource('worker')['args'] = {'common': 'yes'}
def args(num):
for i in range(num):
def f(t):
t.resource('worker')['args']['arg'] = i
yield f
logs = task.parallel_run(3, args(7))
print(json.dumps(list(logs), indent=2))
def main(args):
use_mlboard = False
mlboard = None
if client:
mlboard = client.Client()
try:
mlboard.apps.get()
except Exception:
mlboard = None
print('Do not use mlboard.')
else:
print('Use mlboard parameters logging.')
use_mlboard = True
if args.use_split_dataset:
dataset_tmp = facenet.get_dataset(args.data_dir)
train_set, test_set = split_dataset(dataset_tmp,
args.min_nrof_images_per_class,
args.nrof_train_images_per_class)
if args.mode == 'TRAIN':
dataset = train_set
elif args.mode == 'CLASSIFY':
dataset = test_set
else:
dataset = facenet.get_dataset(args.data_dir)
update_data({'mode': args.mode}, use_mlboard, mlboard)
# Check that there are at least one training image per class
for cls in dataset:
assert len(
cls.image_paths
) > 0, 'There must be at least one image for each class in the dataset'
paths, labels = facenet.get_image_paths_and_labels(dataset)
print('Number of classes: %d' % len(dataset))
print('Number of images: %d' % len(paths))
data = {
'num_classes': len(dataset),
'num_images': len(paths),
'model_path': args.model,
'image_size': args.image_size,
'data_dir': args.data_dir,
'batch_size': args.batch_size,
}
update_data(data, use_mlboard, mlboard)
# Load the model
print('Loading feature extraction model')
xml_file = args.model
bin_file = xml_file[:-3] + 'bin'
net = ie.IENetwork.from_ir(xml_file, bin_file)
extensions = os.environ.get('INTEL_EXTENSIONS_PATH')
plugin = ie.IEPlugin(device=args.device)
if extensions and "CPU" in args.device:
for ext in extensions.split(':'):
print("LOAD extension from {}".format(ext))
plugin.add_cpu_extension(ext)
input_name = list(net.inputs.keys())[0]
output_name = net.outputs[0]
exec_net = plugin.load(net)
# Run forward pass to calculate embeddings
print('Calculating features for images')
nrof_images = len(paths)
nrof_batches_per_epoch = int(math.ceil(1.0 * nrof_images /
args.batch_size))
emb_array = np.zeros((nrof_images, 512))
for i in range(nrof_batches_per_epoch):
start_index = i * args.batch_size
end_index = min((i + 1) * args.batch_size, nrof_images)
paths_batch = paths[start_index:end_index]
for j in range(end_index - start_index):
print('Batch {} <-> {}'.format(paths_batch[j],
labels[start_index + j]))
images = facenet.load_data(paths_batch, False, False, args.image_size)
images = images.transpose([0, 3, 1, 2])
feed_dict = {input_name: images}
output = exec_net.infer(feed_dict)
output = output[output_name]
emb_array[start_index:end_index, :] = output
classifier_filename_exp = os.path.expanduser(args.classifier_filename)
if args.mode == 'TRAIN':
# Train classifier
print('Training classifier')
model = svm.SVC(kernel='linear', probability=True)
model.fit(emb_array, labels)
# Create a list of class names
class_names = [cls.name.replace('_', ' ') for cls in dataset]
print('Classes:')
print(class_names)
# Saving classifier model
with open(classifier_filename_exp, 'wb') as outfile:
pickle.dump((model, class_names), outfile, protocol=2)
print('Saved classifier model to file "%s"' % classifier_filename_exp)
elif args.mode == 'CLASSIFY':
# Classify images
print('Testing classifier')
with open(classifier_filename_exp, 'rb') as infile:
(model, class_names) = pickle.load(infile)
print('Loaded classifier model from file "%s"' %
classifier_filename_exp)
predictions = model.predict_proba(emb_array)
best_class_indices = np.argmax(predictions, axis=1)
best_class_probabilities = predictions[
np.arange(len(best_class_indices)), best_class_indices]
for i in range(len(best_class_indices)):
print('%4d %s: %.3f' % (i, class_names[best_class_indices[i]],
best_class_probabilities[i]))
accuracy = np.mean(np.equal(best_class_indices, labels))
update_data({'accuracy': accuracy}, use_mlboard, mlboard)
print('Accuracy: %.3f' % accuracy)
if args.upload_model and accuracy >= args.upload_threshold:
timestamp = datetime.datetime.now().strftime('%s')
upload_model(use_mlboard, mlboard, classifier_filename_exp,
'facenet-classifier', '1.0.0-%s' % timestamp)
def main():
targs = build_config()
parser = ArgumentParser()
group = parser.add_mutually_exclusive_group(required=True)
group.set_defaults(worker=False)
group.set_defaults(evaluator=False)
group.add_argument('--worker',
dest='worker',
action='store_true',
help='Training')
group.add_argument('--evaluator',
dest='evaluator',
action='store_true',
help='Continuously evaluate model')
parser.add_argument('--training_dir')
parser.add_argument('--research_dir')
parser.add_argument('--build_id')
parser.add_argument('--only_train', default='False')
parser.add_argument('--export', type=str_bool, help='Export model')
parser.add_argument('--model_name')
parser.add_argument('--model_version')
args, _ = parser.parse_known_args()
with open('faster_rcnn.config', 'r') as cf:
data = cf.read()
config_html = '<html><head></head><body><pre style="word-wrap: break-word; white-space: pre-wrap;">{}</pre></body></html>'.format(
data)
client.Client().update_task_info({'#documents.config.html': config_html})
sys.path.append(args.research_dir)
num_steps = targs['num_steps']
model_dir = '{}/{}'.format(args.training_dir, args.build_id)
config = tf.estimator.RunConfig(model_dir=model_dir)
train_and_eval_dict = model_lib.create_estimator_and_inputs(
run_config=config,
hparams=model_hparams.create_hparams(None),
pipeline_config_path='faster_rcnn.config',
train_steps=num_steps,
sample_1_of_n_eval_examples=1,
sample_1_of_n_eval_on_train_examples=(5))
estimator = train_and_eval_dict['estimator']
train_input_fn = train_and_eval_dict['train_input_fn']
train_steps = train_and_eval_dict['train_steps']
eval_input_fns = train_and_eval_dict['eval_input_fns']
if args.evaluator:
tf.logging.info('Starting Evaluation.')
model_name = None
model_version = None
if args.export:
model_name = args.model_name
model_version = args.model_version
continuous_eval(estimator, model_dir, eval_input_fns[0],
'validation_data', args, model_name, model_version)
elif os.environ.get("TF_CONFIG", '') != '':
tf.logging.info('Starting Distributed.')
eval_on_train_input_fn = train_and_eval_dict['eval_on_train_input_fn']
predict_input_fn = train_and_eval_dict['predict_input_fn']
train_spec, eval_specs = model_lib.create_train_and_eval_specs(
train_input_fn,
eval_input_fns,
eval_on_train_input_fn,
predict_input_fn,
train_steps,
eval_on_train_data=False)
tf.estimator.train_and_evaluate(estimator, train_spec, eval_specs[0])
else:
tf.logging.info('Starting Training.')
estimator.train(input_fn=train_input_fn, max_steps=train_steps)
import datetime
import glob
import json
import logging
import os
import shutil
import sys
import time
LOG = logging.getLogger(__name__)
try:
from mlboardclient.api import client
ml_board_client = client.Client()
except:
ml_board_client = None
meta_suff = '.__meta.json'
video_dir = os.environ['DATA_DIR']
def start_processing(config_file, job_id):
if ml_board_client is None:
logging.error('Unable to prepare file: ml_board client is not defined')
return
LOG.info('Run task %s: %s', job_id, config_file)
app = ml_board_client.apps.get()
task = app.task('process')
with open(config_file, 'r') as f:
data = json.load(f)
