def get_image_classese_train_eval(data_path): """get image list when data struct is { |-- data_url |-- train |-- class_1 |-- a.jpg ... |-- class_2 |-- b.jpg ... ... |-- eval |-- class_1 |-- c.jpg ... |-- class_2 |-- d.jpg } :param data_path: data store url Returns: train_data_list, eval_data_list, """ image_label_name = {} image_list_train = [] label_index = 0 class_name = [] # get all labeled train data image_list_set = file.list_directory(os.path.join(data_path, 'train')) assert not image_list_set == [], 'there is no file in data url' for i in image_list_set: if file.is_directory(os.path.join(data_path, 'train', i)): img_list = file.list_directory(os.path.join(data_path, 'train', i)) for j in img_list: label = label_index class_name.append(i) if not '.xml' in j and not '.txt' in j: image_list_train.append([os.path.join(data_path, 'train', i, j), label]) image_label_name[i] = label_index label_index += 1 # get all labeled eval data image_list_eval = [] image_list_set = file.list_directory(os.path.join(data_path, 'eval')) assert not image_list_set == [], 'there is no file in data url' for i in image_list_set: if file.is_directory(os.path.join(data_path, 'eval', i)): img_list = file.list_directory(os.path.join(data_path, 'eval', i)) for j in img_list: label = image_label_name[i] if not '.xml' in j and not '.txt' in j: image_list_eval.append([os.path.join(data_path, 'eval', i, j), label]) return image_list_train, image_list_eval, class_name
def get_image_classese_raw(data_path, split_spec): """get image list when data struct is { |-- data_url |-- class_1 |-- a.jpg |-- b.jpg |-- class_2 |-- c.jpg |-- d.jpg ... |-- label_map_dict (optional) } :param data_path: data store url Returns: train_data_list, eval_data_list, """ image_set = [] class_name = [] # get all labeled train data image_list_set = file.list_directory(data_path) for i in image_list_set: if not file.is_directory(os.path.join(data_path, i)): image_list_set.remove(i) assert not image_list_set == [], 'there is no file in data url' label_index = 0 for i in image_list_set: if file.is_directory(os.path.join(data_path, i)): img_list = file.list_directory(os.path.join(data_path, i)) for j in img_list: label = label_index class_name.append(i) if not '.xml' in j and not '.txt' in j: image_set.append([os.path.join(data_path, i, j), label]) label_index += 1 # split to train and eval image_list_train = [] image_list_eval = [] start_examples = 0 for i in image_list_set: image_list_set = file.list_directory(os.path.join(data_path, i)) num_examples = len(image_list_set) train_num = int(num_examples * split_spec) shuffle_list = list(range(start_examples, start_examples + num_examples)) random.shuffle(shuffle_list) for idx, item in enumerate(shuffle_list): if idx < train_num: image_list_train.append(image_set[item]) else: image_list_eval.append(image_set[item]) start_examples += num_examples return image_list_train, image_list_eval, class_name
def get_image_list(data_path, split_spec): """get image list [[image_path, label_path]] :param data_path: data store url :param split_spec: split train percent if data doesn't have evaluation data Returns: train_data_list, eval_data_list, """ image_list_train = [] image_list_eval = [] class_name = None file_list = file.list_directory(data_path) donot_have_directory = True if 'cache' in file_list: file_list.remove('cache') for i in file_list: if file.is_directory(os.path.join(data_path, i)): donot_have_directory = False break if 'Images' and 'Annotations' in file_list: image_list_train, image_list_eval, class_name = \ get_image_images_annotation(data_path, split_spec) elif 'train' and 'eval' in file_list: file_list = file.list_directory(os.path.join(data_path, 'train')) is_raw = True if 'cache' in file_list: file_list.remove('cache') for i in file_list: if file.is_directory(os.path.join(data_path, 'train', i)): is_raw = False break if 'Images' and 'Annotations' in file_list: image_list_train, image_list_eval = get_image_train_eval(data_path) elif 'image_to_annotation.csv' in file_list: image_list_train, image_list_eval = get_image_csv(data_path) elif is_raw: image_list_train, image_list_eval = \ get_image_train_eval_raw(data_path) else: image_list_train, image_list_eval, class_name = \ get_image_classese_train_eval(data_path) elif donot_have_directory: image_list_train, image_list_eval, class_name = get_image_raw_txt(data_path, split_spec) else: image_list_train, image_list_eval, class_name = get_image_classese_raw(data_path, split_spec) return image_list_train, image_list_eval, class_name
def eval_model(FLAGS): if FLAGS.eval_weights_path != '': if file.is_directory(FLAGS.eval_weights_path): test_batch_h5(FLAGS) else: test_single_h5(FLAGS, FLAGS.eval_weights_path) elif FLAGS.eval_pb_path != '': test_single_model(FLAGS)
def check_args(FLAGS): if FLAGS.mode not in ['train', 'save_pb', 'eval']: raise Exception('FLAGS.mode error, should be train, save_pb or eval') if FLAGS.num_classes == 0: raise Exception( 'FLAGS.num_classes error, ' 'should be a positive number associated with your classification task' ) if FLAGS.mode == 'train': if FLAGS.data_url == '': raise Exception('you must specify FLAGS.data_url') if not file.exists(FLAGS.data_url): raise Exception('FLAGS.data_url: %s is not exist' % FLAGS.data_url) if FLAGS.restore_model_path != '' and (not file.exists( FLAGS.restore_model_path)): raise Exception('FLAGS.restore_model_path: %s is not exist' % FLAGS.restore_model_path) if file.is_directory(FLAGS.restore_model_path): raise Exception( 'FLAGS.restore_model_path must be a file path, not a directory, %s' % FLAGS.restore_model_path) if FLAGS.train_url == '': raise Exception('you must specify FLAGS.train_url') elif not file.exists(FLAGS.train_url): file.make_dirs(FLAGS.train_url) if FLAGS.deploy_script_path != '' and (not file.exists( FLAGS.deploy_script_path)): raise Exception('FLAGS.deploy_script_path: %s is not exist' % FLAGS.deploy_script_path) if FLAGS.deploy_script_path != '' and file.exists(FLAGS.train_url + '/model'): raise Exception( FLAGS.train_url + '/model is already exist, only one model directoty is allowed to exist' ) if FLAGS.test_data_url != '' and (not file.exists( FLAGS.test_data_url)): raise Exception('FLAGS.test_data_url: %s is not exist' % FLAGS.test_data_url) if FLAGS.mode == 'save_pb': if FLAGS.deploy_script_path == '' or FLAGS.freeze_weights_file_path == '': raise Exception( 'you must specify FLAGS.deploy_script_path ' 'and FLAGS.freeze_weights_file_path when you want to save pb') if not file.exists(FLAGS.deploy_script_path): raise Exception('FLAGS.deploy_script_path: %s is not exist' % FLAGS.deploy_script_path) if not file.is_directory(FLAGS.deploy_script_path): raise Exception( 'FLAGS.deploy_script_path must be a directory, not a file path, %s' % FLAGS.deploy_script_path) if not file.exists(FLAGS.freeze_weights_file_path): raise Exception('FLAGS.freeze_weights_file_path: %s is not exist' % FLAGS.freeze_weights_file_path) if file.is_directory(FLAGS.freeze_weights_file_path): raise Exception( 'FLAGS.freeze_weights_file_path must be a file path, not a directory, %s ' % FLAGS.freeze_weights_file_path) if file.exists( FLAGS.freeze_weights_file_path.rsplit('/', 1)[0] + '/model'): raise Exception('a model directory is already exist in ' + FLAGS.freeze_weights_file_path.rsplit('/', 1)[0] + ', please rename or remove the model directory ') if FLAGS.mode == 'eval': if FLAGS.eval_weights_path == '' and FLAGS.eval_pb_path == '': raise Exception( 'you must specify FLAGS.eval_weights_path ' 'or FLAGS.eval_pb_path when you want to evaluate a model') if FLAGS.eval_weights_path != '' and FLAGS.eval_pb_path != '': raise Exception( 'you must specify only one of FLAGS.eval_weights_path ' 'and FLAGS.eval_pb_path when you want to evaluate a model') if FLAGS.eval_weights_path != '' and (not file.exists( FLAGS.eval_weights_path)): raise Exception('FLAGS.eval_weights_path: %s is not exist' % FLAGS.eval_weights_path) if FLAGS.eval_pb_path != '' and (not file.exists(FLAGS.eval_pb_path)): raise Exception('FLAGS.eval_pb_path: %s is not exist' % FLAGS.eval_pb_path) if not file.is_directory(FLAGS.eval_pb_path) or ( not FLAGS.eval_pb_path.endswith('model')): raise Exception( 'FLAGS.eval_pb_path must be a directory named model ' 'which contain saved_model.pb and variables, %s' % FLAGS.eval_pb_path) if FLAGS.test_data_url == '': raise Exception( 'you must specify FLAGS.test_data_url when you want to evaluate a model' ) if not file.exists(FLAGS.test_data_url): raise Exception('FLAGS.test_data_url: %s is not exist' % FLAGS.test_data_url)