def export(checkpoint_dir, params):
logging.info("start export model")
session_config = None
conf = tf.estimator.RunConfig(
model_dir=checkpoint_dir,
session_config=session_config
)
net = gan.DAGANEstimator(
params=params,
model_dir=checkpoint_dir,
config=conf,
)
feature_placeholders = {
'i': tf.placeholder(tf.float32, [params['batch_size'], gan.IMAGE_SIZE[0], gan.IMAGE_SIZE[1], 3],
name='i'),
'z': tf.placeholder(tf.float32, [params['batch_size'], params['z_dim']],
name='z'),
}
receiver = tf.estimator.export.build_raw_serving_input_receiver_fn(feature_placeholders)
export_path = net.export_savedmodel(checkpoint_dir, receiver)
export_path = export_path.decode("utf-8")
client.update_task_info({'model_path': export_path})
def make_openvino(exported_path):
app = mlboard.apps.get()
task = app.tasks.get('openvino')
task.resource('converter')['workdir'] = exported_path
task.resource(
'converter'
)['command'] = 'mo_tf.py --input_model frozen.pb --input_shape [1,64,64,3]'
task.resource('converter')['args'] = {
'input_model': 'frozen.pb',
'input_shape': '[1,64,64,3]',
}
task.start(comment='Convert to OpenVino')
completed = task.wait()
if completed.status != 'Succeeded':
logging.warning("Task %s-%s completed with status %s." %
(completed.name, completed.build, completed.status))
logging.warning('Please take a look at the corresponding task logs'
' for more information about failure.')
logging.warning("Workflow completed with status ERROR")
sys.exit(1)
client.update_task_info(
{'openvino_model': os.path.join(exported_path, 'frozen.xml')})
logging.info("Task %s-%s completed with status %s." %
(completed.name, completed.build, completed.status))
def validate(m, name):
LOG.info("Start validation...")
data = pd.read_csv(os.environ.get('DATA_DIR') + "/test.csv")
all = 0.0
good = 0.0
for index, row in data.iterrows():
x = float(row['x'])
y = float(row['y'])
label = int(row['label'])
resp = m.servings.call(
name,
'svm', {"features": [{
"x": {
"Float": x
},
"y": {
"Float": y
}
}]},
port='9000')
if resp.status_code == 200:
res = json.loads(resp.content)['classes'][0]
if res == label:
good += 1
all += 1
else:
return 'Failed request to serving: %d' % (resp.status_code)
accuracy = good / all
LOG.info("Accuracy %.2d", accuracy)
client.update_task_info({'accuracy': accuracy})
return ''
def make_openvino(exported_path,params):
app = mlboard.apps.get()
task = app.tasks.get('openvino')
task.resource('worker')['workdir'] = exported_path
task.resource('worker')['command'] = 'mo_tf.py'
task.resource('worker')['args'] = {
'input_model': 'frozen.pb',
'input': 'i,z',
'input_shape': '[{},{},{},3],[{},{}]'.format(params['batch_size'],gan.IMAGE_SIZE[0],gan.IMAGE_SIZE[1],params['batch_size'],params['z_dim']),
}
task.start(comment='Convert to OpenVino')
completed = task.wait()
if completed.status != 'Succeeded':
logging.warning(
"Task %s-%s completed with status %s."
% (completed.name, completed.build, completed.status)
)
logging.warning(
'Please take a look at the corresponding task logs'
' for more information about failure.'
)
logging.warning("Workflow completed with status ERROR")
sys.exit(1)
client.update_task_info({'openvino_model': os.path.join(exported_path, 'frozen.xml')})
logging.info(
"Task %s-%s completed with status %s."
% (completed.name, completed.build, completed.status)
)
def main():
parser = get_parser()
args = parser.parse_args()
checked_value = f(args.x)
print('f(%s) = %s' % (args.x, checked_value))
client.update_task_info({'checked_value': float(checked_value)})
def export(checkpoint_dir, task_name, build_id, catalog_name):
featureX = tf.contrib.layers.real_valued_column('x')
featureY = tf.contrib.layers.real_valued_column('y')
model = tf.contrib.learn.SVM(example_id_column='i',
feature_columns=[featureX, featureY],
model_dir=checkpoint_dir)
feature_spec = {
'x': tf.FixedLenFeature(dtype=tf.float32, shape=[1]),
'y': tf.FixedLenFeature(dtype=tf.float32, shape=[1])
}
serving_fn = tf.contrib.learn.utils.input_fn_utils.build_parsing_serving_input_fn(
feature_spec)
export_path = model.export_savedmodel(export_dir_base=checkpoint_dir,
serving_input_fn=serving_fn)
export_path = export_path.decode("utf-8")
logging.info("\nModel Path: %s", export_path)
client.update_task_info({'model_path': export_path},
task_name=task_name,
build_id=build_id)
client.update_task_info({
'model_path': export_path,
'checkpoint_path': checkpoint_dir
})
if catalog_name is not None:
ml = client.Client()
ml.model_upload(catalog_name, '1.0.' + build_id, export_path)
def main():
args = parse_args()
learn = gen_inference_deep(root_folder=pathlib.Path(
os.environ['CODE_DIR']),
weights_name='ColorizeArtistic_gen')
filtr = MasterFilter([ColorizerFilter(learn=learn)],
render_factor=render_factor)
output_dir = os.path.join(os.environ['TRAINING_DIR'],
os.environ['BUILD_ID'])
if not os.path.exists(output_dir):
os.mkdir(output_dir)
output_file = os.path.join(output_dir, 'result.mp4')
try:
client.update_task_info({
'input': args.input,
'output': output_file,
'elsapsed(sec)': 0
})
except:
logging.info('Unexpected error during submit state: {}'.format(
sys.exc_info()[0]))
if args.yt:
import youtube_dl
input_file = os.path.join(output_dir, 'test.mp4')
ydl_opts = {
'format': 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4',
'outtmpl': input_file
}
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
ydl.download([args.input])
else:
input_file = args.input
cap = cv2.VideoCapture(input_file)
fourcc = cv2.VideoWriter_fourcc(*'MP4V')
w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
fps = int(cap.get(cv2.CAP_PROP_FPS))
out = cv2.VideoWriter(output_file, fourcc, fps, (w, h))
frames = 0
while True:
ret, frame = cap.read()
if not ret:
break
frame = Image.fromarray(frame[:, :, ::-1])
frame = filtr.filter(frame, frame, render_factor=render_factor)
out.write(np.array(frame)[:, :, ::-1])
frames += 1
if frames % fps == 0:
logging.info('Processing frame: {}'.format(frames))
try:
client.update_task_info({'elsapsed(sec)': int(frames / fps)})
except:
logging.info('Unexpected error during submit state: {}'.format(
sys.exc_info()[0]))
out.release()
def main():
args = parse_args()
h2o.init(ip=args.host, port=args.port)
# Upload the prostate dataset that comes included in the h2o python package
prostate = h2o.load_dataset("prostate")
# Print a description of the prostate data
prostate.describe()
# Randomly split the dataset into ~70/30, training/test sets
client.update_task_info({
'test_train': 0.7,
'learn_rate': 0.2,
})
train, test = prostate.split_frame(ratios=[0.70])
# Convert the response columns to factors (for binary classification problems)
train["CAPSULE"] = train["CAPSULE"].asfactor()
test["CAPSULE"] = test["CAPSULE"].asfactor()
# Build a (classification) GLM
from h2o.estimators import H2OGradientBoostingEstimator
prostate_gbm = H2OGradientBoostingEstimator(distribution="bernoulli",
ntrees=10,
max_depth=8,
min_rows=10,
learn_rate=0.2)
prostate_gbm.train(x=["AGE", "RACE", "PSA", "VOL", "GLEASON"],
y="CAPSULE",
training_frame=train)
# Show the model
prostate_gbm.show()
# Predict on the test set and show the first ten predictions
predictions = prostate_gbm.predict(test)
predictions.show()
# Fetch a tree, print number of tree nodes, show root node description
from h2o.tree import H2OTree, H2ONode
tree = H2OTree(prostate_gbm, 0, "0")
tree.root_node.show()
# Show default performance metrics
performance = prostate_gbm.model_performance(test)
performance.show()
client.update_task_info({
'mse': performance.mse(),
'rmse': performance.rmse(),
'auc': performance.auc(),
'gini': performance.gini(),
'logloss': performance.logloss(),
})
def train(dataset_dir, checkpoint_dir, l1_regularization, l2_regularization,
steps, needExport):
train_set = tf.data.TextLineDataset(dataset_dir + '/train.csv')
validation_set = tf.data.TextLineDataset(dataset_dir + '/test.csv')
logging.info("start build svm model")
with tf.Session():
def _parse(line):
row = tf.string_split([line], delimiter=',').values
i = row[0]
x = tf.string_to_number(row[1], tf.float32)
y = tf.string_to_number(row[2], tf.float32)
label = tf.string_to_number(row[3], tf.int32)
return {
'i': tf.reshape(i, [1]),
'x': tf.reshape(x, [1]),
'y': tf.reshape(y, [1])
}, tf.reshape(label, [1])
train_set = train_set.skip(1).map(_parse).shuffle(100).repeat()
validation_set = validation_set.skip(1).map(_parse)
def _input_fn():
return train_set.make_one_shot_iterator().get_next()
def _validation_fn():
return validation_set.make_one_shot_iterator().get_next()
featureX = tf.contrib.layers.real_valued_column('x')
featureY = tf.contrib.layers.real_valued_column('y')
model = tf.contrib.learn.SVM(example_id_column='i',
feature_columns=[featureX, featureY],
model_dir=checkpoint_dir,
l1_regularization=l1_regularization,
l2_regularization=l2_regularization)
model.fit(input_fn=_input_fn, steps=steps)
metrics = model.evaluate(input_fn=_validation_fn)
logging.info("Done:\nValidation Loss: %.4f\nValidation Accuracy: %.4f",
metrics['loss'], metrics['accuracy'])
client.update_task_info({
'accuracy': float(metrics['accuracy']),
'loss': float(metrics['loss'])
})
if needExport:
feature_spec = {
'x': tf.FixedLenFeature(dtype=tf.float32, shape=[1]),
'y': tf.FixedLenFeature(dtype=tf.float32, shape=[1])
}
#serving_fn = tf.estimator.export.build_parsing_serving_input_receiver_fn(feature_spec)
serving_fn = tf.contrib.learn.utils.input_fn_utils.build_parsing_serving_input_fn(
feature_spec)
export_path = model.export_savedmodel(export_dir_base=checkpoint_dir,
serving_input_fn=serving_fn)
export_path = export_path.decode("utf-8")
logging.info("\nModel Path: %s", export_path)
client.update_task_info({'model_path': export_path})
def main(args):
params = {
'num_pools': args.num_pools,
'num_epochs': args.num_epochs,
'drop_prob': args.drop_prob,
'num_chans': args.num_chans,
'batch_size': args.batch_size,
'lr': args.lr,
'lr_step_size': args.lr_step_size,
'lr_gamma': args.lr_gamma,
'weight_decay': args.weight_decay,
'checkpoint': str(args.checkpoint_dir),
'save_summary_steps': args.save_summary_steps,
'save_checkpoints_secs': args.save_checkpoints_secs,
'save_checkpoints_steps': args.save_checkpoints_steps,
'keep_checkpoint_max': args.keep_checkpoint_max,
'log_step_count_steps': args.log_step_count_steps,
'warm_start_from': args.warm_start_from,
'use_seed': False,
'resolution': args.resolution,
'data_set': args.data_set,
'optimizer': args.optimizer,
'loss': args.loss,
'coco': args.coco,
'features': [int(i) for i in args.features.split(',')]
}
if args.export:
export(args.checkpoint_dir, params)
return
if not tf.gfile.Exists(args.checkpoint_dir):
tf.gfile.MakeDirs(args.checkpoint_dir)
if args.worker:
client.update_task_info({
'num-pools': args.num_pools,
'drop-prob': args.drop_prob,
'num-chans': args.num_chans,
'batch-size': args.batch_size,
'lr.lr': args.lr,
'lr.lr-step-size': args.lr_step_size,
'lr.lr-gamma': args.lr_gamma,
'weight-decay': args.weight_decay,
'checkpoint_path': str(args.checkpoint_dir),
'resolution': args.resolution,
'features': args.features,
})
train('train', args.checkpoint_dir, params)
else:
cluster = {'chief': ['fake_worker1:2222'],
'ps': ['fake_ps:2222'],
'worker': ['fake_worker2:2222']}
os.environ['TF_CONFIG'] = json.dumps(
{
'cluster': cluster,
'task': {'type': 'evaluator', 'index': 0}
})
train('eval', args.checkpoint_dir, params)
def main(_):
if FLAGS.mode == 'export':
log_dir = os.environ.get('TRAINING_DIR') + '/' + FLAGS.build
client.update_task_info({'checkpoint_path': log_dir})
export_path = os.path.join(log_dir, str(FLAGS.model_version))
export(export_path, log_dir)
client.update_task_info({'model_path': export_path}, task_name='train', build_id=FLAGS.build)
if FLAGS.catalog_name is not None:
ml = client.Client()
ml.model_upload(FLAGS.catalog_name, '1.0.' + FLAGS.build, export_path)
else:
train()
def export(checkpoint_dir, params):
m = client.Client()
base_id = '0'
if os.environ.get('BASE_TASK_BUILD_ID', '') != '':
app = m.apps.get()
base_id = os.environ['BASE_TASK_BUILD_ID']
task_name = os.environ['BASE_TASK_NAME']
task = app.get_task(task_name, base_id)
checkpoint_dir = task.exec_info['checkpoint_path']
params['num_chans'] = task.exec_info['num-chans']
params['num_pools'] = task.exec_info['num-pools']
params['resolution'] = task.exec_info['resolution']
params['checkpoint'] = checkpoint_dir
conf = tf.estimator.RunConfig(
model_dir=checkpoint_dir,
)
params['batch_size'] = 1
features_def = [int(i) for i in task.exec_info.get('features','3').split(',')]
logging.info('Features Def: {}'.format(features_def))
params['features'] = features_def
feature_placeholders = {
'image': tf.placeholder(tf.float32, [1, None, None, sum(features_def)], name='image'),
}
receiver = tf.estimator.export.build_raw_serving_input_receiver_fn(feature_placeholders)
net = FastBGNet(
params=params,
model_dir=checkpoint_dir,
config=conf,
)
models = os.path.join(checkpoint_dir, 'models')
build_id = os.environ['BUILD_ID']
export_dir = os.path.join(models, build_id)
os.makedirs(export_dir, exist_ok=True)
export_path = net.export_savedmodel(
export_dir,
receiver,
)
export_path = export_path.decode("utf-8")
base = os.path.basename(export_path)
driver_data = {'driver': 'tensorflow', 'path': base}
with open(os.path.join(export_dir, '_model_config.yaml'), 'w') as f:
yaml.dump(driver_data, f)
params['num_chans'] = task.exec_info['num-chans']
params['num_pools'] = task.exec_info['num-pools']
params['resolution'] = task.exec_info['resolution']
version = f'1.{base_id}.{build_id}'
model_name = 'person-mask'
m.model_upload(model_name, version, export_dir, spec=serving_spec())
client.update_task_info({'model_path': export_path, 'num-chans': params['num_chans'],
'features':','.join([str(i) for i in features_def]),
'num-pools': params['num_pools'], 'resolution': params['resolution'],
'model_reference': catalog_ref(model_name, 'mlmodel', version)})
def test(checkpoint_dir, params):
logging.info("start test model")
session_config = None
conf = tf.estimator.RunConfig(
model_dir=checkpoint_dir,
session_config=session_config
)
net = gan.DAGANEstimator(
params=params,
model_dir=checkpoint_dir,
config=conf,
)
feature_placeholders = {
'i': tf.placeholder(tf.float32, [params['batch_size'], gan.IMAGE_SIZE[0], gan.IMAGE_SIZE[1], 3],
name='i'),
'z': tf.placeholder(tf.float32, [params['batch_size'], params['z_dim']],
name='z'),
}
input_fn = gan.test_fn(params)
predictions = net.predict(input_fn)
num_generations = params['num_generations']
h = []
images = []
for p in predictions:
p = (p + 1) / 2 * 255
p = np.uint8(np.clip(p, 0, 255))
h.append(p)
if len(h) == num_generations:
images.append(np.concatenate(h, axis=1))
h = []
if len(images) == num_generations:
break
images = np.concatenate(images, axis=0)
im = PIL.Image.fromarray(images)
im.save(checkpoint_dir + '/result.png')
with io.BytesIO() as output:
im.save(output, format='PNG')
contents = output.getvalue()
rpt = '<html><img src="data:image/png;base64,{}"/></html>'.format(base64.b64encode(contents).decode())
mlboard.update_task_info({'#documents.result.html': rpt})
receiver = tf.estimator.export.build_raw_serving_input_receiver_fn(feature_placeholders)
export_path = net.export_savedmodel(checkpoint_dir, receiver)
export_path = export_path.decode("utf-8")
client.update_task_info({'model_path': export_path})
freeze(export_path,params,True)
def main():
parser = get_parser()
args = parser.parse_args()
if args.debug:
tf.logging.set_verbosity(tf.logging.DEBUG)
else:
tf.logging.set_verbosity(tf.logging.INFO)
if not tf.gfile.Exists(args.checkpoint_dir):
tf.gfile.MakeDirs(args.checkpoint_dir)
if args.export_model:
export(args.checkpoint_dir, args.task_name, args.build_id,
args.catalog_name)
else:
client.update_task_info({'checkpoint_path': args.checkpoint_dir})
train(args.dataset_dir, args.checkpoint_dir, args.l1_regularization,
args.l2_regularization, args.steps, args.export)
def export(export_path, log_dir):
model_path = os.path.join(log_dir, "model.ckpt")
logging.info('Exporting model to %s ...', export_path)
with tf.Graph().as_default(), tf.Session() as sess:
batch_image_str_placeholder = tf.placeholder(
dtype=tf.string,
shape=[None],
name='encoded_image_string_tensor'
)
def decode(encoded_image_string_tensor):
image_tensor = tf.image.decode_image(encoded_image_string_tensor,
channels=1)
image_tensor.set_shape((None, None, 1))
image_tensor = tf.image.resize_image_with_crop_or_pad(image_tensor, 28, 28)
return image_tensor
x_image = tf.map_fn(
decode,
elems=batch_image_str_placeholder,
dtype=tf.uint8,
parallel_iterations=32,
back_prop=False
)
x_image = tf.to_float(x_image) / 255.0
keep_prob = tf.constant(1, dtype=tf.float32)
y_conv, _ = deepnn_builder(x_image, keep_prob)
tensor_info_images = tf.saved_model.utils.build_tensor_info(batch_image_str_placeholder)
tensor_info_results = tf.saved_model.utils.build_tensor_info(y_conv)
saver = tf.train.Saver()
logging.info('Restoring model from %s ...', model_path)
saver.restore(sess, model_path)
prediction_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={'images': tensor_info_images},
outputs={'y_conv': tensor_info_results},
method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME))
builder = tf.saved_model.builder.SavedModelBuilder(export_path)
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING],
signature_def_map={
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
prediction_signature
})
builder.save()
client.update_task_info({'model_path': export_path})
logging.info("Model exported to %s", export_path)
def main():
parser = get_parser()
args = parser.parse_args()
if args.debug:
logging.root.setLevel('DEBUG')
m = client.Client()
app = m.apps.get()
task = app.task('train')
if args.data_version!='':
task.config['datasetRevisions']=[{'volumeName': 'data', 'revision': args.data_version}]
task.resource('worker')['command'] = 'python svm.py --steps=1000 --checkpoint_dir=$TRAINING_DIR/$BUILD_ID'
spec = (optimizator.ParamSpecBuilder().resource('worker')
.param('l2_regularization')
.bounds(1,10)
.param('l1_regularization')
.bounds(1,10)
.build())
LOG.info('Run with param spec = %s', spec)
result = task.optimize(
'accuracy',
spec,
init_steps=args.init_steps,
iterations=args.iterations,
method=args.method,
max_parallel=args.parallel,
direction='maximize'
)
best = result['best']
LOG.info('Found best build %s:%s: %.2f', best.name,best.build,best.exec_info['accuracy'])
client.update_task_info({'checkpoint_path':best.exec_info['checkpoint_path'],
'accuracy':best.exec_info['accuracy'],'build':best.build})
LOG.info('Exporting model %s...',best.build)
export = app.task('export')
export.resource('run')['command'] = 'python svm.py --export_model'
export.resource('run')['args']= {
'catalog_name': 'my_svm_model',
'task_name': best.name,
'build_id': best.build,
'checkpoint_dir': best.exec_info['checkpoint_path']
}
export.start()
export.wait()
client.update_task_info({'model_path':export.exec_info['model_path']})
def export(checkpoint_dir, params):
conf = tf.estimator.RunConfig(model_dir=checkpoint_dir, )
feature_placeholders = {
'images':
tf.placeholder(tf.float32, [params['batch_size'], 299, 299, 3],
name='images'),
}
receiver = tf.estimator.export.build_raw_serving_input_receiver_fn(
feature_placeholders, default_batch_size=params['batch_size'])
net = model.Model(
params=params,
model_dir=checkpoint_dir,
config=conf,
)
export_path = net.export_savedmodel(
checkpoint_dir,
receiver,
)
export_path = export_path.decode("utf-8")
client.update_task_info({'model_path': export_path})
def freeze(exported_path,params,do_openvino=True):
from tensorflow.python.saved_model import loader
from tensorflow.python.saved_model import signature_constants as tf_const
sess = tf.Session(graph=tf.Graph())
graph = loader.load(sess, [tf.saved_model.tag_constants.SERVING], exported_path)
outputs = graph.signature_def.get(tf_const.DEFAULT_SERVING_SIGNATURE_DEF_KEY).outputs
toutputs = []
for out in list(outputs.values()):
name = sess.graph.get_tensor_by_name(out.name).name
p = name.split(':')
name = p[0] if len(p) > 0 else name
toutputs += [name]
output_graph_def = tf.graph_util.convert_variables_to_constants(
sess, sess.graph.as_graph_def(), toutputs)
out_path = os.path.join(exported_path, 'frozen.pb')
client.update_task_info({'frozen_graph': out_path})
with tf.gfile.GFile(out_path, "wb") as f:
f.write(output_graph_def.SerializeToString())
if do_openvino:
make_openvino(exported_path,params)
return out_path
def main():
parser = get_parser()
args = parser.parse_args()
dataset = '/tmp/data'
if not os.path.exists(dataset):
os.makedirs(dataset)
train = pd.DataFrame(random.randint(low=0, high=100, size=(1000, 2)),
columns=['x', 'y'])
train['label'] = train.apply(lambda v: 0 if v['x'] > v['y'] +
(5 - random.random_sample() * 10) else 1,
axis=1)
test = pd.DataFrame(random.randint(low=0, high=100, size=(100, 2)),
columns=['x', 'y'])
test['label'] = test.apply(lambda v: 0 if v['x'] > v['y'] else 1, axis=1)
train.to_csv(dataset + '/train.csv')
test.to_csv(dataset + '/test.csv')
kl = client.Client()
kl.datasets.push(os.environ.get('WORKSPACE_NAME'),
args.dataset,
args.version,
dataset,
create=True)
client.update_task_info(
{'dataset': '%s:%s' % (args.dataset, args.version)})
def train():
# Import data
logging.info('Loading dataset...')
mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True, source_url=FLAGS.source_url)
logging.info('Initing tf graph...')
# Create the model
x = tf.placeholder(tf.float32, [None, 784], name="x")
# Define loss and optimizer
y_ = tf.placeholder(tf.float32, [None, 10], name="y_")
# Build the graph for the deep net
y_conv, keep_prob = deepnn(x)
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))
tf.summary.scalar('cross_entropy', cross_entropy)
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy', accuracy)
with tf.Session() as sess:
merged = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(FLAGS.log_dir,
sess.graph)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
for i in range(FLAGS.training_iteration):
batch = mnist.train.next_batch(50)
summary, _ = sess.run([merged, train_step],
feed_dict={x: batch[0], y_: batch[1], keep_prob: FLAGS.drop_out})
if i % 100 == 0:
train_accuracy = sess.run(accuracy, feed_dict={
x: batch[0], y_: batch[1], keep_prob: 1.0})
train_writer.add_summary(summary, i)
client.update_task_info({'train_accuracy': float(train_accuracy)})
logging.info('Step %d, training accuracy %g', i, train_accuracy)
saver.save(sess, os.path.join(FLAGS.log_dir, "model.ckpt"))
client.update_task_info({'checkpoint_path': FLAGS.log_dir})
test_accuracy = accuracy.eval(feed_dict={
x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0})
logging.info('Test accuracy %g', test_accuracy)
client.update_task_info({'test_accuracy': float(test_accuracy)})
if FLAGS.model_version > 0:
export(os.path.join(FLAGS.log_dir, str(FLAGS.model_version)), FLAGS.log_dir)
def submit(params):
if os.environ.get('PROJECT_ID', None):
from mlboardclient.api import client
client.update_task_info(params)
def main(args):
output_dir = os.path.expanduser(args.output_dir)
bounding_boxes_filename = os.path.join(output_dir, 'bounding_boxes.txt')
align_filename = os.path.join(output_dir, 'align.pkl')
align_data_args = dict(vars(args))
# the next arguments can be changed w/o changing aligned images
del align_data_args['complementary']
del align_data_args['input_dir']
del align_data_args['output_dir']
align_data = {}
clear_output_dir = True
if args.complementary:
if os.path.isfile(align_filename):
print_fun("Check previous align data")
with open(align_filename, 'rb') as infile:
(align_data_args_loaded,
align_data_loaded) = pickle.load(infile)
if align_data_args == align_data_args_loaded:
print_fun("Loaded data about %d aligned classes" %
len(align_data_loaded))
align_data = align_data_loaded
clear_output_dir = False
else:
print_fun(
"Previous align data is for another arguments, skipped"
)
if clear_output_dir:
print_fun("Clearing output dir")
shutil.rmtree(output_dir, ignore_errors=True)
if not os.path.isdir(output_dir):
print_fun("Creating output dir")
os.makedirs(output_dir)
# Store some git revision info in a text file in the log directory
src_path, _ = os.path.split(os.path.realpath(__file__))
# facenet.store_revision_info(src_path, output_dir, ' '.join(sys.argv))
dataset = facenet.get_dataset(args.input_dir)
print_fun('Creating networks and loading parameters')
# Load driver
drv = driver.load_driver("openvino")
# Instantinate driver
serving = drv()
serving.load_model(
args.face_detection_path,
device="CPU",
flexible_batch_size=True,
)
bg_rm_drv = bg_remove.get_driver(args.bg_remove_path)
input_name = list(serving.inputs.keys())[0]
output_name = list(serving.outputs.keys())[0]
threshold = 0.5
min_face_area = args.min_face_size**2
with open(bounding_boxes_filename, "w") as text_file:
nrof_images_total = 0
nrof_successfully_aligned = 0
for cls in dataset:
output_class_dir = os.path.join(output_dir, cls.name)
output_class_dir_created = False
if cls.name in align_data:
align_data_class = align_data[cls.name]
else:
align_data_class = {}
for image_path in cls.image_paths:
nrof_images_total += 1
filename = os.path.splitext(os.path.split(image_path)[1])[0]
output_filename = os.path.join(output_class_dir,
filename + '.png')
if not os.path.exists(output_filename):
try:
img = cv2.imread(image_path,
cv2.IMREAD_COLOR).astype(np.float32)
except Exception as e:
error_message = '{}: {}'.format(image_path, e)
print_fun('ERROR: %s' % error_message)
continue
img_hash = hashlib.sha1(img.tostring()).hexdigest()
if image_path in align_data_class and align_data_class[
image_path] == img_hash:
print_fun("%s - cached" % image_path)
continue
align_data_class[image_path] = hashlib.sha1(
img.tostring()).hexdigest()
print_fun(image_path)
if len(img.shape) <= 2:
print_fun('WARNING: Unable to align "%s", shape %s' %
(image_path, img.shape))
text_file.write('%s\n' % output_filename)
continue
if bg_rm_drv is not None:
img = bg_rm_drv.apply_mask(img)
serving_img = cv2.resize(img, (300, 300),
interpolation=cv2.INTER_AREA)
serving_img = np.transpose(serving_img, [2, 0, 1]).reshape(
[1, 3, 300, 300])
raw = serving.predict({input_name: serving_img
})[output_name].reshape([-1, 7])
# 7 values:
# class_id, label, confidence, x_min, y_min, x_max, y_max
# Select boxes where confidence > factor
bboxes_raw = raw[raw[:, 2] > threshold]
bboxes_raw[:, 3] = bboxes_raw[:, 3] * img.shape[1]
bboxes_raw[:, 5] = bboxes_raw[:, 5] * img.shape[1]
bboxes_raw[:, 4] = bboxes_raw[:, 4] * img.shape[0]
bboxes_raw[:, 6] = bboxes_raw[:, 6] * img.shape[0]
bounding_boxes = np.zeros([len(bboxes_raw), 5])
bounding_boxes[:, 0:4] = bboxes_raw[:, 3:7]
bounding_boxes[:, 4] = bboxes_raw[:, 2]
# Get the biggest box: find the box with largest square:
# (y1 - y0) * (x1 - x0) - size of box.
bbs = bounding_boxes
area = (bbs[:, 3] - bbs[:, 1]) * (bbs[:, 2] - bbs[:, 0])
if len(area) < 1:
print_fun('WARNING: Unable to align "%s", n_faces=%s' %
(image_path, len(area)))
text_file.write('%s\n' % output_filename)
continue
num = np.argmax(area)
if area[num] < min_face_area:
print_fun(
'WARNING: Face found but too small - about {}px '
'width against required minimum of {}px. Try'
' adjust parameter --min-face-size'.format(
int(np.sqrt(area[num])), args.min_face_size))
continue
bounding_boxes = np.stack([bbs[num]])
imgs = openvino_detection.get_images(
img,
bounding_boxes,
face_crop_size=args.image_size,
face_crop_margin=args.margin,
prewhiten=False,
)
for i, cropped in enumerate(imgs):
nrof_successfully_aligned += 1
bb = bounding_boxes[i]
filename_base, file_extension = os.path.splitext(
output_filename)
output_filename_n = "{}_{}{}".format(
filename_base, i, file_extension)
text_file.write(
'%s %d %d %d %d\n' %
(output_filename_n, bb[0], bb[1], bb[2], bb[3]))
if not output_class_dir_created:
output_class_dir_created = True
if not os.path.exists(output_class_dir):
os.makedirs(output_class_dir)
cv2.imwrite(output_filename_n, cropped)
align_data[cls.name] = align_data_class
with open(align_filename, 'wb') as align_file:
pickle.dump((align_data_args, align_data), align_file, protocol=2)
print_fun('Total number of images: %d' % nrof_images_total)
print_fun('Number of successfully aligned images: %d' %
nrof_successfully_aligned)
build_id = os.environ.get('BUILD_ID', None)
if os.environ.get('PROJECT_ID', None) and (build_id is not None):
from mlboardclient.api import client
client.update_task_info({'aligned_location': output_dir})
from mlboardclient.api import client
import sys
data = ""
for line in sys.stdin:
data += line
if data != "":
client.update_task_info({'#documents.report.html':data})
def train(mode, checkpoint_dir, params):
logging.info("start build model")
save_summary_steps = params['save_summary_steps']
save_checkpoints_secs = params['save_checkpoints_secs'] if params[
'save_checkpoints_steps'] is None else None
save_checkpoints_steps = params['save_checkpoints_steps']
gpu = os.environ.get('GPUXX_Y', None)
if gpu is None:
session_config = None
else:
session_config = tf.ConfigProto()
session_config.gpu_options.visible_device_list = gpu
conf = tf.estimator.RunConfig(
model_dir=checkpoint_dir,
save_summary_steps=save_summary_steps,
save_checkpoints_secs=save_checkpoints_secs,
save_checkpoints_steps=save_checkpoints_steps,
keep_checkpoint_max=params['keep_checkpoint_max'],
log_step_count_steps=params['log_step_count_steps'],
session_config=session_config)
net = gan.INFOGAN(
params=params,
model_dir=checkpoint_dir,
config=conf,
)
logging.info("Start %s mode", mode)
if mode == 'train':
_, input_fn = gan.input_fn(params, True)
net.train(input_fn=input_fn)
logging.info('Export model')
feature_placeholders = {
'image':
tf.placeholder(tf.float32,
[1, gan.IMAGE_SIZE[0], gan.IMAGE_SIZE[1], 3],
name='image_placeholder')
}
receiver = tf.estimator.export.build_raw_serving_input_receiver_fn(
feature_placeholders)
export_path = net.export_savedmodel(checkpoint_dir, receiver)
export_path = export_path.decode("utf-8")
client.update_task_info({'model_path': export_path})
logging.info('Build index to {}'.format(export_path))
frozen = freeze(export_path, params['openvino'])
logging.info('Save frozen graph: {}'.format(frozen))
params['epoch'] = 1
input_set, input_fn = gan.input_fn(params, False)
predictions = net.predict(input_fn=input_fn)
i = 0
clip = 1e-3
features = []
for p in predictions:
np.clip(p, -clip, clip, p)
i += 1
features.append(p)
if i == len(input_set):
break
features = np.stack(features)
with open(export_path + '/features.pkl', 'wb') as output:
pickle.dump(features, output, pickle.HIGHEST_PROTOCOL)
pickle.dump(input_set, output, pickle.HIGHEST_PROTOCOL)
version = '1.{}.0'.format(os.environ['BUILD_ID'])
mlboard.model_upload('infogan-similarity', version, export_path)
client.update_task_info(
{'model': catalog_ref('infogan-similarity', 'mlmodel', version)})
logging.info(
"New model uploaded as 'infogan-similarity', version '%s'." %
(version))
elif mode == 'eval':
train_fn = gan.null_dataset()
train_spec = tf.estimator.TrainSpec(input_fn=train_fn)
eval_fn = gan.eval_fn()
eval_spec = tf.estimator.EvalSpec(input_fn=eval_fn,
steps=1,
start_delay_secs=10,
throttle_secs=10)
tf.estimator.train_and_evaluate(net, train_spec, eval_spec)
else:
logging.info("Not implemented")
def main():
parser = get_parser()
args = parser.parse_args()
# BATCH_SIZE must be multiple of <executor.cores>:
# in this case multiple of 3: 3,6,9,12 etc.
if args.batch_size % args.executor_cores != 0:
raise RuntimeError(
'batch size must be multiple of <executor-cores> parameter!'
)
cores = args.executor_cores
batch_size = args.batch_size
conf = (
common.create_spark_conf()
.setAppName('pyspark-mnist')
.setMaster(args.master)
)
conf = conf.set('spark.executor.cores', cores)
conf = conf.set('spark.cores.max', cores)
conf.set("spark.jars",os.environ.get('BIGDL_JARS'))
LOG.info('initialize with spark conf:')
sc = pyspark.SparkContext(conf=conf)
common.init_engine()
LOG.info('initialize training RDD:')
##Files from kuberlab dataset
files = glob.glob(os.environ.get('DATA_DIR')+'/train/*.png')
LOG.info('Train size: %d',len(files))
def mapper(x):
label = int(x.split('/')[-1].split('-')[-1][:-4])+1
image = imageio.imread('file://'+x).astype(np.float32).reshape(1,28,28)/255
return common.Sample.from_ndarray(image, label)
train_rdd = sc.parallelize(files).map(mapper)
opt = optimizer.Optimizer(
model=build_model(10),
training_rdd=train_rdd,
criterion=criterion.ClassNLLCriterion(),
optim_method=optimizer.SGD(
learningrate=0.01, learningrate_decay=0.0002
),
end_trigger=optimizer.MaxEpoch(args.epoch),
batch_size=batch_size
)
trained_model = opt.optimize()
LOG.info("training finished")
LOG.info('saving model...')
path = args.output_dir
if not os.path.exists(path):
os.makedirs(path)
trained_model.saveModel(
path + '/model.pb',
path + '/model.bin',
over_write=True
)
client.update_task_info({'checkpoint_path': path,'model_path': path})
LOG.info('successfully saved!')
files = glob.glob(os.environ.get('DATA_DIR')+'/test/*.png')
LOG.info('Validation size: %d',len(files))
test_rdd = sc.parallelize(files).map(mapper)
results = trained_model.evaluate(test_rdd, batch_size , [optimizer.Top1Accuracy()])
accuracy = results[0].result
client.update_task_info({'test_accuracy': float(accuracy)})
sc.stop()
