def receive_training_data(): start_time = time.time() print('API called ' + str(request.path) + ' : ' + str(request.form)) if 'file' not in request.files: raise exc.InvalidData('No file part', status_code=404) if not request.form['PID'] or not request.form['Category'] or not request.form['Label']: raise exc.InvalidData('Missing metadata', status_code=404) saved_files = [] uploaded_files = request.files.getlist('file') print('uploaded files : ' + str(uploaded_files)) for file in uploaded_files: if file.filename == '': raise exc.InvalidData('No selected file', status_code=404) if not file or not utils.allowed_file(file.filename): raise exc.InvalidData('Image Data invalid', status_code=404) filename = secure_filename(file.filename) filepath = utils.create_filepath(request.form, app.config['UPLOAD_FOLDER'], filename) file.save(filepath) saved_files.append(filepath) DirPath = os.path.dirname(filepath) controller.insert_record(request.form['PID'], request.form['Category'], request.form['Label'], DirPath) inserted_id = controller.insert_model(request.form['PID'], request.form['Category']) controller.update_model_status(request.form['PID'], request.form['Category'], controller.TrainingStatus.not_trained) if app.config['LOG_DETAIL']: log_misc = {} log_misc["ImagesFilePath"] = [] for saved_file in saved_files: log_misc["ImagesFilePath"].append(saved_file) time_elapsed = time.time() - start_time controller.insert_log_flask_api(request.path, request.form, time_elapsed, misc=log_misc) return "Images Uploaded!"
def run_inference_multi(): start_time = time.time() print('API called ' + str(request.path) + ' : ' + str(request.form)) if 'file' not in request.files: raise exc.InvalidData('No file part', status_code=404) saved_files = [] upload_dir = str(uuid.uuid4()) uploaded_files = request.files.getlist('file') print('uploaded files : ' + str(uploaded_files)) for file in uploaded_files: if file.filename == '': raise exc.InvalidData('No selected file', status_code=404) if not file or not utils.allowed_file(file.filename): raise exc.InvalidData('Image Data invalid', status_code=404) filename = secure_filename(file.filename) filename = os.path.join(upload_dir, filename) filepath = utils.create_filepath(request.form, app.config['INFERENCE_FOLDER'], filename) print('Saved image file for inference : ' + filepath) file.save(filepath) saved_files.append(filepath) async_result = queue_inference.apply_async(queue='queue_inference', args=[request.form['PID'], request.form['Category'], None, saved_files]) result = async_result.get(interval=0.005) if not app.config['LOG_DETAIL']: shutil.rmtree(uplod_dir) if app.config['LOG_DETAIL']: time_elapsed = time.time() - start_time log_misc = {} log_misc["ImagesFilePath"] = saved_files train_mode = controller.get_training_mode(request.form['PID'], request.form['Category']) if train_mode is not None and 'NetworkModel' in train_mode: log_misc["NetworkModel"] = train_mode["NetworkModel"] if train_mode is not None and 'TrainingMode' in train_mode: log_misc["TrainingMode"] = train_mode["TrainingMode"] if train_mode is not None and 'GlobalStepCount' in train_mode: log_misc["GlobalStepCount"] = train_mode["GlobalStepCount"] log_result = {} for key in result: encode_key = key.replace("\\", "\\\\").replace("\$", "\\u0024").replace(".", "\\u002e") log_result[encode_key] = result[key] controller.insert_log_flask_api(request.path, request.form, time_elapsed, response=log_result, misc=log_misc) if result is not None: return make_response(jsonify(result)) else: raise exc.InvalidData('No model available for the specified data.', status_code=404)
def send_model(): start_time = time.time() print('API called ' + str(request.path) + ' : ' + str(request.form)) if 'PID' not in request.form or 'Category' not in request.form: raise exc.InvalidData('Missing Data for training', status_code=404) output = controller.get_model_path(request.form['PID'], request.form['Category']) if app.config['LOG_DETAIL']: time_elapsed = time.time() - start_time controller.insert_log_flask_api(request.path, request.form, time_elapsed, response=output) if output: return make_response(jsonify(output)) else: raise exc.InvalidData('No model available for the specified data.', status_code=404)
def queue_training(pid, category, quick_train): # set as non-daemon process to create sub process from this celery task # https://github.com/celery/celery/issues/1709 current_process()._config['daemon'] = False task_id = queue_training.request.id print('Queue training is called. PID=' + pid + ', Category=' + category + ', task ID=' + task_id + ', process ID=' + str(os.getpid())) # update training flag to cancel task controller.upsert_train_task(pid, category, task_id) controller.upsert_train_task_process(task_id, os.getpid()) # start training try: output_graph_filepath = utils.create_model_filepath(pid, category, app.config['MODEL_FOLDER']) if output_graph_filepath: controller.update_model_status(pid, category, controller.TrainingStatus.training) output_model_dir = utils.create_model_filepath(pid, category, app.config['MODEL_FOLDER']) train_thread = TrainThread(app.config['UPLOAD_FOLDER'], app.config['UNKNOWN_FOLDER'], app.config['PRETRAINED_FOLDER'], app.config['TENSORFLOW_FOLDER'], app.config['APNS_SETTINGS']) train_thread.run_train(pid, category, app.config['NETWORK_MODEL'], task_id, output_model_dir, quick_train=quick_train) else: raise exc.InvalidData("Couldn't create output graph file path.", status_code=404) except Exception: traceback.print_exc() controller.update_model_status(pid, category, controller.TrainingStatus.error) controller.delete_train_task_process(task_id, os.getpid()) return controller.delete_train_task_process(task_id, os.getpid()) return
def _start_training(request_form): print('API called /api/train : ' + str(request_form)) if 'PID' not in request_form or 'Category' not in request_form: raise exc.InvalidData('Missing Data for training', status_code=404) quick_train = False if 'QuickTrain' in request_form: quick_train = True # cancel old train tasks print('[start_training] Start cancel training...') old_train_task_ids = cancel_training(request_form['PID'], request_form['Category']) print('[start_training] Finish cancel training') try : # add training process to queue print('[start_training] Start add training queue...') result = queue_training.apply_async(queue='queue_train', args=[request_form['PID'], request_form['Category'], quick_train]) print('[start_training] Finish add training queue.') # update devices that wait push message if 'BundleID' in request_form and len(request_form['BundleID'])>0 and 'DeviceToken' in request_form and len(request_form['DeviceToken'])>0: print('Added train wait device : task=' + result.task_id + ', bunlde ID=' + request_form['BundleID'] + ', device=' + request_form['DeviceToken']) controller.upsert_train_wait_device(result.task_id, request_form['BundleID'], request_form['DeviceToken']) for old_train_task_id in old_train_task_ids: old_train_wait_bundle_ids, old_train_wait_devices = controller.get_train_wait_devices(old_train_task_id) for idx in range(len(old_train_wait_bundle_ids)): controller.delete_train_wait_device(old_train_task_id, old_train_wait_bundle_ids[idx], old_train_wait_devices[idx]) controller.upsert_train_wait_device(result.task_id, old_train_wait_bundle_ids[idx], old_train_wait_devices[idx]) print('Added training queue : PID=' + request_form['PID'] + ', Category=' + request_form['Category'] + ', task ID=' + result.task_id + ', process ID=' + str(os.getpid())) except AlreadyQueued: print('Already queued : PID=' + request_form['PID'] + ', Category=' + request_form['Category'] + ', process ID=' + str(os.getpid()))
def get_training_status(): start_time = time.time() print('API called ' + str(request.path) + ' : ' + str(request.form)) if 'PID' not in request.form or 'Category' not in request.form: raise exc.InvalidData('Missing Data for training status', status_code=404) training_status = controller.get_training_status(request.form['PID'], request.form['Category']) if app.config['LOG_DETAIL']: time_elapsed = time.time() - start_time controller.insert_log_flask_api(request.path, request.form, time_elapsed, response=training_status) if training_status is None: raise exc.InvalidData('No model trained for this data', status_code=404) else: print('training status : ' + str(training_status)) return make_response(jsonify(training_status))
def get_split(split_name, dataset_dir, file_pattern, file_pattern_for_counting, labels_to_name, num_classes, items_to_descriptions): if split_name not in ['train', 'validation']: raise exc.InvalidData('Invalid split name for training', status_code=404) # file path for tfrecord_files file_pattern_path = os.path.join(dataset_dir, file_pattern % (split_name)) # get number of examples num_samples = 0 file_pattern_for_counting = file_pattern_for_counting + '_' + split_name tfrecords_to_count = [ os.path.join(dataset_dir, file) for file in os.listdir(dataset_dir) if file.startswith(file_pattern_for_counting) ] for tfrecord_file in tfrecords_to_count: for record in tf.python_io.tf_record_iterator(tfrecord_file): num_samples += 1 # create keys_to_features for the decoder keys_to_features = { 'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''), 'image/format': tf.FixedLenFeature((), tf.string, default_value='jpg'), 'image/class/label': tf.FixedLenFeature([], tf.int64, default_value=tf.zeros([], dtype=tf.int64)), } # create items_to_handlers for the decoder. items_to_handlers = { 'image': slim.tfexample_decoder.Image(), 'label': slim.tfexample_decoder.Tensor('image/class/label'), } # create dataset reader = tf.TFRecordReader decoder = slim.tfexample_decoder.TFExampleDecoder(keys_to_features, items_to_handlers) dataset = slim.dataset.Dataset(data_sources=file_pattern_path, decoder=decoder, reader=reader, num_readers=4, num_samples=num_samples, num_classes=num_classes, labels_to_name=labels_to_name, items_to_descriptions=items_to_descriptions) return dataset
def send_model_file(pid, category, filename): start_time = time.time() print('API called ' + str(request.path) + ' : ' + str(request.form)) if app.config['LOG_DETAIL']: time_elapsed = time.time() - start_time controller.insert_log_flask_api(request.path, request.form, time_elapsed) try: path = os.path.join(pid, category, filename) return send_file(os.path.join(app.config['MODEL_FOLDER'], path)) except Exception as e: raise exc.InvalidData('Failed to send file', status_code=404)
def update_label(): start_time = time.time() print('API called ' + str(request.path) + ' : ' + str(request.form)) if 'PID' not in request.form or 'Category' not in request.form or 'Label' not in request.form or 'NewLabel' not in request.form: raise exc.InvalidData('Missing Data for training', status_code=404) controller.update_label(request.form['PID'], request.form['Category'], request.form['Label'], request.form['NewLabel']) if app.config['LOG_DETAIL']: time_elapsed = time.time() - start_time controller.insert_log_flask_api(request.path, request.form, time_elapsed) return "Label Updated!"
def list_labels(): start_time = time.time() print('API called ' + str(request.path) + ' : ' + str(request.form)) if 'PID' not in request.form or 'Category' not in request.form: raise exc.InvalidData('Missing Data for training', status_code=404) result = controller.list_labels(request.form['PID'], request.form['Category']) if app.config['LOG_DETAIL']: time_elapsed = time.time() - start_time controller.insert_log_flask_api(request.path, request.form, time_elapsed, response=result) return make_response(jsonify(result))
def delete_label(): start_time = time.time() print('API called ' + str(request.path) + ' : ' + str(request.form)) if app.config['LOG_DETAIL']: # keep deleting image if detail log option is set keep_deleted_image_dirpath = utils.create_dirpath(request.form, app.config['DELETED_IMAGE_FOLDER']) status = controller.delete_label(request.form['PID'], request.form['Category'], request.form['Label'], keep_deleted_image_directory=keep_deleted_image_dirpath) else: status = controller.delete_label(request.form['PID'], request.form['Category'], request.form['Label']) if app.config['LOG_DETAIL']: time_elapsed = time.time() - start_time controller.insert_log_flask_api(request.path, request.form, time_elapsed, response=status) if status: return "Label Deleted" else: raise exc.InvalidData('Error deleting label. Please check input data.', status_code=404)