def receive_training_data():
start_time = time.time()
print('API called ' + str(request.path) + ' : ' + str(request.form))
if 'file' not in request.files:
raise exc.InvalidData('No file part', status_code=404)
if not request.form['PID'] or not request.form['Category'] or not request.form['Label']:
raise exc.InvalidData('Missing metadata', status_code=404)
saved_files = []
uploaded_files = request.files.getlist('file')
print('uploaded files : ' + str(uploaded_files))
for file in uploaded_files:
if file.filename == '':
raise exc.InvalidData('No selected file', status_code=404)
if not file or not utils.allowed_file(file.filename):
raise exc.InvalidData('Image Data invalid', status_code=404)
filename = secure_filename(file.filename)
filepath = utils.create_filepath(request.form, app.config['UPLOAD_FOLDER'], filename)
file.save(filepath)
saved_files.append(filepath)
DirPath = os.path.dirname(filepath)
controller.insert_record(request.form['PID'], request.form['Category'], request.form['Label'], DirPath)
inserted_id = controller.insert_model(request.form['PID'], request.form['Category'])
controller.update_model_status(request.form['PID'], request.form['Category'], controller.TrainingStatus.not_trained)
if app.config['LOG_DETAIL']:
log_misc = {}
log_misc["ImagesFilePath"] = []
for saved_file in saved_files:
log_misc["ImagesFilePath"].append(saved_file)
time_elapsed = time.time() - start_time
controller.insert_log_flask_api(request.path, request.form, time_elapsed, misc=log_misc)
return "Images Uploaded!"
def run_inference_multi():
start_time = time.time()
print('API called ' + str(request.path) + ' : ' + str(request.form))
if 'file' not in request.files:
raise exc.InvalidData('No file part', status_code=404)
saved_files = []
upload_dir = str(uuid.uuid4())
uploaded_files = request.files.getlist('file')
print('uploaded files : ' + str(uploaded_files))
for file in uploaded_files:
if file.filename == '':
raise exc.InvalidData('No selected file', status_code=404)
if not file or not utils.allowed_file(file.filename):
raise exc.InvalidData('Image Data invalid', status_code=404)
filename = secure_filename(file.filename)
filename = os.path.join(upload_dir, filename)
filepath = utils.create_filepath(request.form, app.config['INFERENCE_FOLDER'], filename)
print('Saved image file for inference : ' + filepath)
file.save(filepath)
saved_files.append(filepath)
async_result = queue_inference.apply_async(queue='queue_inference', args=[request.form['PID'], request.form['Category'], None, saved_files])
result = async_result.get(interval=0.005)
if not app.config['LOG_DETAIL']:
shutil.rmtree(uplod_dir)
if app.config['LOG_DETAIL']:
time_elapsed = time.time() - start_time
log_misc = {}
log_misc["ImagesFilePath"] = saved_files
train_mode = controller.get_training_mode(request.form['PID'], request.form['Category'])
if train_mode is not None and 'NetworkModel' in train_mode:
log_misc["NetworkModel"] = train_mode["NetworkModel"]
if train_mode is not None and 'TrainingMode' in train_mode:
log_misc["TrainingMode"] = train_mode["TrainingMode"]
if train_mode is not None and 'GlobalStepCount' in train_mode:
log_misc["GlobalStepCount"] = train_mode["GlobalStepCount"]
log_result = {}
for key in result:
encode_key = key.replace("\\", "\\\\").replace("\$", "\\u0024").replace(".", "\\u002e")
log_result[encode_key] = result[key]
controller.insert_log_flask_api(request.path, request.form, time_elapsed, response=log_result, misc=log_misc)
if result is not None:
return make_response(jsonify(result))
else:
raise exc.InvalidData('No model available for the specified data.', status_code=404)
def send_model():
start_time = time.time()
print('API called ' + str(request.path) + ' : ' + str(request.form))
if 'PID' not in request.form or 'Category' not in request.form:
raise exc.InvalidData('Missing Data for training', status_code=404)
output = controller.get_model_path(request.form['PID'], request.form['Category'])
if app.config['LOG_DETAIL']:
time_elapsed = time.time() - start_time
controller.insert_log_flask_api(request.path, request.form, time_elapsed, response=output)
if output:
return make_response(jsonify(output))
else:
raise exc.InvalidData('No model available for the specified data.', status_code=404)
def queue_training(pid, category, quick_train):
# set as non-daemon process to create sub process from this celery task
# https://github.com/celery/celery/issues/1709
current_process()._config['daemon'] = False
task_id = queue_training.request.id
print('Queue training is called. PID=' + pid + ', Category=' + category + ', task ID=' + task_id + ', process ID=' + str(os.getpid()))
# update training flag to cancel task
controller.upsert_train_task(pid, category, task_id)
controller.upsert_train_task_process(task_id, os.getpid())
# start training
try:
output_graph_filepath = utils.create_model_filepath(pid, category, app.config['MODEL_FOLDER'])
if output_graph_filepath:
controller.update_model_status(pid, category, controller.TrainingStatus.training)
output_model_dir = utils.create_model_filepath(pid, category, app.config['MODEL_FOLDER'])
train_thread = TrainThread(app.config['UPLOAD_FOLDER'], app.config['UNKNOWN_FOLDER'], app.config['PRETRAINED_FOLDER'],
app.config['TENSORFLOW_FOLDER'], app.config['APNS_SETTINGS'])
train_thread.run_train(pid, category, app.config['NETWORK_MODEL'], task_id, output_model_dir, quick_train=quick_train)
else:
raise exc.InvalidData("Couldn't create output graph file path.", status_code=404)
except Exception:
traceback.print_exc()
controller.update_model_status(pid, category, controller.TrainingStatus.error)
controller.delete_train_task_process(task_id, os.getpid())
return
controller.delete_train_task_process(task_id, os.getpid())
return
def _start_training(request_form):
print('API called /api/train : ' + str(request_form))
if 'PID' not in request_form or 'Category' not in request_form:
raise exc.InvalidData('Missing Data for training', status_code=404)
quick_train = False
if 'QuickTrain' in request_form:
quick_train = True
# cancel old train tasks
print('[start_training] Start cancel training...')
old_train_task_ids = cancel_training(request_form['PID'], request_form['Category'])
print('[start_training] Finish cancel training')
try :
# add training process to queue
print('[start_training] Start add training queue...')
result = queue_training.apply_async(queue='queue_train', args=[request_form['PID'], request_form['Category'], quick_train])
print('[start_training] Finish add training queue.')
# update devices that wait push message
if 'BundleID' in request_form and len(request_form['BundleID'])>0 and 'DeviceToken' in request_form and len(request_form['DeviceToken'])>0:
print('Added train wait device : task=' + result.task_id + ', bunlde ID=' + request_form['BundleID'] + ', device=' + request_form['DeviceToken'])
controller.upsert_train_wait_device(result.task_id, request_form['BundleID'], request_form['DeviceToken'])
for old_train_task_id in old_train_task_ids:
old_train_wait_bundle_ids, old_train_wait_devices = controller.get_train_wait_devices(old_train_task_id)
for idx in range(len(old_train_wait_bundle_ids)):
controller.delete_train_wait_device(old_train_task_id, old_train_wait_bundle_ids[idx], old_train_wait_devices[idx])
controller.upsert_train_wait_device(result.task_id, old_train_wait_bundle_ids[idx], old_train_wait_devices[idx])
print('Added training queue : PID=' + request_form['PID'] + ', Category=' + request_form['Category'] + ', task ID=' + result.task_id + ', process ID=' + str(os.getpid()))
except AlreadyQueued:
print('Already queued : PID=' + request_form['PID'] + ', Category=' + request_form['Category'] + ', process ID=' + str(os.getpid()))
def get_training_status():
start_time = time.time()
print('API called ' + str(request.path) + ' : ' + str(request.form))
if 'PID' not in request.form or 'Category' not in request.form:
raise exc.InvalidData('Missing Data for training status', status_code=404)
training_status = controller.get_training_status(request.form['PID'], request.form['Category'])
if app.config['LOG_DETAIL']:
time_elapsed = time.time() - start_time
controller.insert_log_flask_api(request.path, request.form, time_elapsed, response=training_status)
if training_status is None:
raise exc.InvalidData('No model trained for this data', status_code=404)
else:
print('training status : ' + str(training_status))
return make_response(jsonify(training_status))
def get_split(split_name, dataset_dir, file_pattern, file_pattern_for_counting,
labels_to_name, num_classes, items_to_descriptions):
if split_name not in ['train', 'validation']:
raise exc.InvalidData('Invalid split name for training',
status_code=404)
# file path for tfrecord_files
file_pattern_path = os.path.join(dataset_dir, file_pattern % (split_name))
# get number of examples
num_samples = 0
file_pattern_for_counting = file_pattern_for_counting + '_' + split_name
tfrecords_to_count = [
os.path.join(dataset_dir, file) for file in os.listdir(dataset_dir)
if file.startswith(file_pattern_for_counting)
]
for tfrecord_file in tfrecords_to_count:
for record in tf.python_io.tf_record_iterator(tfrecord_file):
num_samples += 1
# create keys_to_features for the decoder
keys_to_features = {
'image/encoded':
tf.FixedLenFeature((), tf.string, default_value=''),
'image/format':
tf.FixedLenFeature((), tf.string, default_value='jpg'),
'image/class/label':
tf.FixedLenFeature([],
tf.int64,
default_value=tf.zeros([], dtype=tf.int64)),
}
# create items_to_handlers for the decoder.
items_to_handlers = {
'image': slim.tfexample_decoder.Image(),
'label': slim.tfexample_decoder.Tensor('image/class/label'),
}
# create dataset
reader = tf.TFRecordReader
decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features,
items_to_handlers)
dataset = slim.dataset.Dataset(data_sources=file_pattern_path,
decoder=decoder,
reader=reader,
num_readers=4,
num_samples=num_samples,
num_classes=num_classes,
labels_to_name=labels_to_name,
items_to_descriptions=items_to_descriptions)
return dataset
def send_model_file(pid, category, filename):
start_time = time.time()
print('API called ' + str(request.path) + ' : ' + str(request.form))
if app.config['LOG_DETAIL']:
time_elapsed = time.time() - start_time
controller.insert_log_flask_api(request.path, request.form, time_elapsed)
try:
path = os.path.join(pid, category, filename)
return send_file(os.path.join(app.config['MODEL_FOLDER'], path))
except Exception as e:
raise exc.InvalidData('Failed to send file', status_code=404)
def update_label():
start_time = time.time()
print('API called ' + str(request.path) + ' : ' + str(request.form))
if 'PID' not in request.form or 'Category' not in request.form or 'Label' not in request.form or 'NewLabel' not in request.form:
raise exc.InvalidData('Missing Data for training', status_code=404)
controller.update_label(request.form['PID'], request.form['Category'], request.form['Label'], request.form['NewLabel'])
if app.config['LOG_DETAIL']:
time_elapsed = time.time() - start_time
controller.insert_log_flask_api(request.path, request.form, time_elapsed)
return "Label Updated!"
def list_labels():
start_time = time.time()
print('API called ' + str(request.path) + ' : ' + str(request.form))
if 'PID' not in request.form or 'Category' not in request.form:
raise exc.InvalidData('Missing Data for training', status_code=404)
result = controller.list_labels(request.form['PID'], request.form['Category'])
if app.config['LOG_DETAIL']:
time_elapsed = time.time() - start_time
controller.insert_log_flask_api(request.path, request.form, time_elapsed, response=result)
return make_response(jsonify(result))
def delete_label():
start_time = time.time()
print('API called ' + str(request.path) + ' : ' + str(request.form))
if app.config['LOG_DETAIL']:
# keep deleting image if detail log option is set
keep_deleted_image_dirpath = utils.create_dirpath(request.form, app.config['DELETED_IMAGE_FOLDER'])
status = controller.delete_label(request.form['PID'], request.form['Category'], request.form['Label'],
keep_deleted_image_directory=keep_deleted_image_dirpath)
else:
status = controller.delete_label(request.form['PID'], request.form['Category'], request.form['Label'])
if app.config['LOG_DETAIL']:
time_elapsed = time.time() - start_time
controller.insert_log_flask_api(request.path, request.form, time_elapsed, response=status)
if status:
return "Label Deleted"
else:
raise exc.InvalidData('Error deleting label. Please check input data.', status_code=404)
