def _get_label_names(self, index_file):
classes = []
if index_file is not None and file.exists(index_file):
label_file = h5py.File(index_file, 'r')
classes_name = label_file['labels_list'][:]
label_file.close()
classes = [name.decode('utf-8') for name in classes_name]
else:
for data_file in self.img_list:
annotation_file = data_file[1]
tree = ET.ElementTree()
parser = ET.XMLParser(target=ET.TreeBuilder())
parser.feed(file.read(annotation_file, binary=True))
tree._root = parser.close()
objs = tree.findall('object')
non_diff_objs = [
obj for obj in objs if int(obj.find('difficult').text) == 0
]
objs = non_diff_objs
for obj in objs:
class_name = obj.find('name').text.lower().strip()
if class_name not in classes:
classes.append(class_name)
if index_file is not None:
index_file_path = index_file
else:
index_file_path = INDEX_FILE_NAME
if file.exists(index_file_path):
file.remove(index_file_path)
label_file = h5py.File(index_file, 'w')
label_file.create_dataset('labels_list',
data=[item.encode() for item in classes])
label_file.close()
return classes
def get_image_train_eval(data_path):
"""get image list when data struct is
{
|-- data_url
|-- train
|-- Images
|-- a.jpg
...
|-- Annotations
|-- a.txt (or a.xml)
|-- label_map_dict (optional)
|-- eval
|-- Images
|-- b.jpg
...
|-- Annotations
|-- b.txt (or b.xml)
...
|-- label_map_dict (optional)
|-- label_map_dict (optional)
}
:param data_path: data store url
Returns:
train_data_list,
eval_data_list,
"""
image_list_train = []
# get all labeled train data
image_list_set = file.list_directory(os.path.join(data_path, 'train', 'Images'))
assert not image_list_set == [], 'there is no file in data url'
for i in image_list_set:
if file.exists(os.path.join(data_path, 'train', 'Annotations', os.path.splitext(i)[0] + '.xml')):
image_list_train.append([os.path.join(data_path, 'train', 'Images', i),
os.path.join(data_path, 'train', 'Annotations', os.path.splitext(i)[0] + '.xml')])
elif file.exists(os.path.join(data_path, 'train', 'Annotations', os.path.splitext(i)[0] + '.txt')):
image_list_train.append([os.path.join(data_path, 'train', 'Images', i),
file.read(os.path.join(data_path, 'train',
'Annotations',
os.path.splitext(i)[0] + '.txt'))])
# get all labeled eval data
image_list_eval = []
image_list_set = []
image_list_set = file.list_directory(os.path.join(data_path, 'eval', 'Images'))
assert not image_list_set == [], 'there is no file in data url'
for i in image_list_set:
if file.exists(os.path.join(data_path, 'eval', 'Annotations', os.path.splitext(i)[0] + '.xml')):
image_list_eval.append([os.path.join(data_path, 'eval', 'Images', i),
os.path.join(data_path, 'eval', 'Annotations', os.path.splitext(i)[0] + '.xml')])
elif file.exists(os.path.join(data_path, 'eval', 'Annotations', os.path.splitext(i)[0] + '.txt')):
image_list_eval.append([os.path.join(data_path, 'eval', 'Images', i),
file.read(os.path.join(data_path, 'eval',
'Annotations',
os.path.splitext(i)[0] + '.txt'))])
return image_list_train, image_list_eval
def get_image_images_annotation(data_path, split_spec):
"""get image list when data struct is
{
|-- data_url
|-- Images
|-- a.jpg
|-- b.jpg
...
|-- Annotations
|-- a.txt (or a.xml)
|-- b.txt (or b.xml)
...
|-- label_map_dict (optional)
}
:param data_path: data store url
:param split_spec: split train percent if data doesn't have evaluation data
Returns:
train_data_list,
eval_data_list,
"""
image_set = []
label_dict = {}
label_num = 0
class_name = []
# get all labeled data
image_list_set = file.list_directory(os.path.join(data_path, 'Images'))
assert not image_list_set == [], 'there is no file in data url'
for i in image_list_set:
if file.exists(os.path.join(data_path, 'Annotations', os.path.splitext(i)[0] + '.xml')):
image_set.append([os.path.join(data_path, 'Images', i),
os.path.join(data_path, 'Annotations', os.path.splitext(i)[0] + '.xml')])
elif file.exists(os.path.join(data_path, 'Annotations', os.path.splitext(i)[0] + '.txt')):
label_name = file.read(os.path.join(data_path, 'Annotations',
os.path.splitext(i)[0] + '.txt'))
if label_name not in label_dict.keys():
label_dict[label_name] = label_num
class_name.append(label_name)
label_num = label_num + 1
image_set.append([os.path.join(data_path, 'Images', i),
label_dict[label_name]])
# split data to train and eval
num_examples = len(image_set)
train_num = int(num_examples * split_spec)
shuffle_list = list(range(num_examples))
random.shuffle(shuffle_list)
image_list_train = []
image_list_eval = []
for idx, item in enumerate(shuffle_list):
if idx < train_num:
image_list_train.append(image_set[item])
else:
image_list_eval.append(image_set[item])
return image_list_train, image_list_eval, class_name
def train_model(FLAGS):
# data flow generator
train_sequence, validation_sequence = data_flow(FLAGS.data_local, FLAGS.batch_size,
FLAGS.num_classes, FLAGS.input_size)
optimizer = adam(lr=FLAGS.learning_rate, clipnorm=0.001)
objective = 'binary_crossentropy'
metrics = ['accuracy']
model = model_fn(FLAGS, objective, optimizer, metrics)
if FLAGS.restore_model_path != '' and file.exists(FLAGS.restore_model_path):
if FLAGS.restore_model_path.startswith('s3://'):
restore_model_name = FLAGS.restore_model_path.rsplit('/', 1)[1]
file.copy(FLAGS.restore_model_path, '/cache/tmp/' + restore_model_name)
model.load_weights('/cache/tmp/' + restore_model_name,by_name=True)
os.remove('/cache/tmp/' + restore_model_name)
else:
model.load_weights(FLAGS.restore_model_path,by_name=True)
if not os.path.exists(FLAGS.train_local):
os.makedirs(FLAGS.train_local)
tensorBoard = TensorBoard(log_dir=FLAGS.train_local)
history = LossHistory(FLAGS)
model.fit_generator(
train_sequence,
steps_per_epoch=len(train_sequence),
epochs=FLAGS.max_epochs,
verbose=1,
callbacks=[history, tensorBoard],
validation_data=validation_sequence,
max_queue_size=10,
workers=int(multiprocessing.cpu_count() * 0.7),
use_multiprocessing=True,
shuffle=True
)
print('training done!')
if FLAGS.deploy_script_path != '':
from save_model import save_pb_model
save_pb_model(FLAGS, model)
if FLAGS.test_data_url != '':
print('test dataset predicting...')
from eval import load_test_data
img_names, test_data, test_labels = load_test_data(FLAGS)
predictions = model.predict(test_data, verbose=0)
right_count = 0
for index, pred in enumerate(predictions):
predict_label = np.argmax(pred, axis=0)
test_label = test_labels[index]
if predict_label == test_label:
right_count += 1
accuracy = right_count / len(img_names)
print('accuracy: %0.4f' % accuracy)
metric_file_name = os.path.join(FLAGS.train_local, 'metric.json')
metric_file_content = '{"total_metric": {"total_metric_values": {"accuracy": %0.4f}}}' % accuracy
with open(metric_file_name, "w") as f:
f.write(metric_file_content + '\n')
print('end')
def save_pb_model(FLAGS, model):
if FLAGS.mode == 'train':
pb_save_dir_local = FLAGS.train_local
pb_save_dir_obs = FLAGS.train_url
elif FLAGS.mode == 'save_pb':
freeze_weights_file_dir = FLAGS.freeze_weights_file_path.rsplit(
'/', 1)[0]
if freeze_weights_file_dir.startswith('s3://'):
pb_save_dir_local = '/cache/tmp'
pb_save_dir_obs = freeze_weights_file_dir
else:
pb_save_dir_local = freeze_weights_file_dir
pb_save_dir_obs = pb_save_dir_local
signature = tf.saved_model.signature_def_utils.predict_signature_def(
inputs={'input_img': model.input},
outputs={'output_score': model.output})
builder = tf.saved_model.builder.SavedModelBuilder(
os.path.join(pb_save_dir_local, 'model'))
legacy_init_op = tf.group(tf.tables_initializer(), name='legacy_init_op')
builder.add_meta_graph_and_variables(
sess=backend.get_session(),
tags=[tf.saved_model.tag_constants.SERVING],
signature_def_map={
'predict_images': signature,
},
legacy_init_op=legacy_init_op)
builder.save()
print('save pb to local path success')
if pb_save_dir_obs.startswith('s3://'):
file.copy_parallel(os.path.join(pb_save_dir_local, 'model'),
os.path.join(pb_save_dir_obs, 'model'))
print('copy pb to %s success' % pb_save_dir_obs)
file.copy(os.path.join(FLAGS.deploy_script_path, 'config.json'),
os.path.join(pb_save_dir_obs, 'model/config.json'))
file.copy(os.path.join(FLAGS.deploy_script_path, 'customize_service.py'),
os.path.join(pb_save_dir_obs, 'model/customize_service.py'))
if file.exists(os.path.join(pb_save_dir_obs, 'model/config.json')) and \
file.exists(os.path.join(pb_save_dir_obs, 'model/customize_service.py')):
print('copy config.json and customize_service.py success')
else:
print('copy config.json and customize_service.py failed')
def check_args(FLAGS):
if FLAGS.mode not in ['train', 'save_pb', 'eval']:
raise Exception('FLAGS.mode error, should be train, save_pb or eval')
if FLAGS.num_classes == 0:
raise Exception(
'FLAGS.num_classes error, '
'should be a positive number associated with your classification task'
)
if FLAGS.mode == 'train':
if FLAGS.data_url == '':
raise Exception('you must specify FLAGS.data_url')
if not file.exists(FLAGS.data_url):
raise Exception('FLAGS.data_url: %s is not exist' % FLAGS.data_url)
if FLAGS.restore_model_path != '' and (not file.exists(
FLAGS.restore_model_path)):
raise Exception('FLAGS.restore_model_path: %s is not exist' %
FLAGS.restore_model_path)
if file.is_directory(FLAGS.restore_model_path):
raise Exception(
'FLAGS.restore_model_path must be a file path, not a directory, %s'
% FLAGS.restore_model_path)
if FLAGS.train_url == '':
raise Exception('you must specify FLAGS.train_url')
elif not file.exists(FLAGS.train_url):
file.make_dirs(FLAGS.train_url)
if FLAGS.deploy_script_path != '' and (not file.exists(
FLAGS.deploy_script_path)):
raise Exception('FLAGS.deploy_script_path: %s is not exist' %
FLAGS.deploy_script_path)
if FLAGS.deploy_script_path != '' and file.exists(FLAGS.train_url +
'/model'):
raise Exception(
FLAGS.train_url +
'/model is already exist, only one model directoty is allowed to exist'
)
if FLAGS.test_data_url != '' and (not file.exists(
FLAGS.test_data_url)):
raise Exception('FLAGS.test_data_url: %s is not exist' %
FLAGS.test_data_url)
if FLAGS.mode == 'save_pb':
if FLAGS.deploy_script_path == '' or FLAGS.freeze_weights_file_path == '':
raise Exception(
'you must specify FLAGS.deploy_script_path '
'and FLAGS.freeze_weights_file_path when you want to save pb')
if not file.exists(FLAGS.deploy_script_path):
raise Exception('FLAGS.deploy_script_path: %s is not exist' %
FLAGS.deploy_script_path)
if not file.is_directory(FLAGS.deploy_script_path):
raise Exception(
'FLAGS.deploy_script_path must be a directory, not a file path, %s'
% FLAGS.deploy_script_path)
if not file.exists(FLAGS.freeze_weights_file_path):
raise Exception('FLAGS.freeze_weights_file_path: %s is not exist' %
FLAGS.freeze_weights_file_path)
if file.is_directory(FLAGS.freeze_weights_file_path):
raise Exception(
'FLAGS.freeze_weights_file_path must be a file path, not a directory, %s '
% FLAGS.freeze_weights_file_path)
if file.exists(
FLAGS.freeze_weights_file_path.rsplit('/', 1)[0] + '/model'):
raise Exception('a model directory is already exist in ' +
FLAGS.freeze_weights_file_path.rsplit('/', 1)[0] +
', please rename or remove the model directory ')
if FLAGS.mode == 'eval':
if FLAGS.eval_weights_path == '' and FLAGS.eval_pb_path == '':
raise Exception(
'you must specify FLAGS.eval_weights_path '
'or FLAGS.eval_pb_path when you want to evaluate a model')
if FLAGS.eval_weights_path != '' and FLAGS.eval_pb_path != '':
raise Exception(
'you must specify only one of FLAGS.eval_weights_path '
'and FLAGS.eval_pb_path when you want to evaluate a model')
if FLAGS.eval_weights_path != '' and (not file.exists(
FLAGS.eval_weights_path)):
raise Exception('FLAGS.eval_weights_path: %s is not exist' %
FLAGS.eval_weights_path)
if FLAGS.eval_pb_path != '' and (not file.exists(FLAGS.eval_pb_path)):
raise Exception('FLAGS.eval_pb_path: %s is not exist' %
FLAGS.eval_pb_path)
if not file.is_directory(FLAGS.eval_pb_path) or (
not FLAGS.eval_pb_path.endswith('model')):
raise Exception(
'FLAGS.eval_pb_path must be a directory named model '
'which contain saved_model.pb and variables, %s' %
FLAGS.eval_pb_path)
if FLAGS.test_data_url == '':
raise Exception(
'you must specify FLAGS.test_data_url when you want to evaluate a model'
)
if not file.exists(FLAGS.test_data_url):
raise Exception('FLAGS.test_data_url: %s is not exist' %
FLAGS.test_data_url)
def get_data_iter(data_path, train_file=None, val_file=None, split_spec=1,
hyper_train={}, hyper_val={}, **kwargs):
train_set = None
val_set = None
train_list = None
val_list = None
if train_file is not None:
assert file.exists(train_file), 'not found train file'
train_path = file.read(train_file).split("\n")[0:-1]
train_list = [path.replace('\r', '').split(' ') for path in train_path]
train_list = [[os.path.join(data_path, path[0]),
os.path.join(data_path, path[1])] for path in train_list]
if val_file is not None:
assert file.exists(val_file), 'not found val file'
val_path = file.read(val_file).split("\n")[0:-1]
val_list = [path.replace('\r', '').split(' ') for path in val_path]
val_list = [[os.path.join(data_path, path[0]),
os.path.join(data_path, path[1])] for path in val_list]
if train_file is None and val_file is None:
train_list, val_list, _ = get_image_list(data_path, split_spec)
if 'anchors' not in kwargs:
kwargs['anchors'] = [[116, 90, 156, 198, 373, 326],
[30, 61, 62, 45, 59, 119],
[10, 13, 16, 30, 33, 23]]
if 'offsets' not in kwargs:
kwargs['offsets'] = [(13, 13), (26, 26), (52, 52)]
if train_list is not None and len(train_list) > 0:
dataset = Detection_dataset(img_list=train_list,
index_file=hyper_train.get(
'index_file', None),
width=hyper_train.get('width', 416),
height=hyper_train.get('height', 416),
is_train=True,
** kwargs)
max_gt_box_number = max([len(item) for item in dataset.label_cache])
batch_size = hyper_train.get('batch_size', 32)
train_set = gluon.data.DataLoader(
dataset=dataset,
batch_size=batch_size,
shuffle=hyper_train.get('shuffle', True),
batchify_fn=_train_batchify_fn(max_gt_box_number),
last_batch='rollover',
num_workers=hyper_train.get('preprocess_threads', 4))
next_data_batch = next(iter(train_set))
setattr(train_set, 'reset', _reset)
setattr(train_set, 'provide_data', _get_provide_data(next_data_batch))
setattr(train_set, 'provide_label', _get_provide_label(
next_data_batch, (batch_size, max_gt_box_number, 4), is_train=True))
if val_list is not None and len(val_list) > 0:
assert 'index_file' in hyper_val and file.exists(
hyper_val['index_file']), 'not found label name file'
dataset = Detection_dataset(img_list=val_list,
index_file=hyper_val.get(
'index_file'),
width=hyper_val.get('width', 416),
height=hyper_val.get('height', 416),
is_train=False,
** kwargs)
max_gt_box_number = max([len(item) for item in dataset.label_cache])
batch_size = hyper_val.get('batch_size', 32)
val_set = gluon.data.DataLoader(
dataset=dataset,
batch_size=batch_size,
shuffle=hyper_val.get('shuffle', True),
batchify_fn=_val_batchify_fn(max_gt_box_number),
last_batch='keep',
num_workers=hyper_val.get('preprocess_threads', 4))
next_data_batch = next(iter(val_set))
setattr(val_set, 'reset', _reset)
setattr(val_set, 'provide_data', _get_provide_data(next_data_batch))
setattr(val_set, 'provide_label', _get_provide_label(
next_data_batch, is_train=False))
return train_set, val_set
def train_model(FLAGS):
# data flow generator
train_sequence, validation_sequence = data_flow(FLAGS.data_local,
FLAGS.batch_size,
FLAGS.num_classes,
FLAGS.input_size)
# optimizer = adam(lr=FLAGS.learning_rate, clipnorm=0.001)
optimizer = Nadam(lr=FLAGS.learning_rate,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
schedule_decay=0.004)
# optimizer = SGD(lr=FLAGS.learning_rate, momentum=0.9)
objective = 'categorical_crossentropy'
metrics = ['accuracy']
model = model_fn(FLAGS, objective, optimizer, metrics)
if FLAGS.restore_model_path != '' and file.exists(
FLAGS.restore_model_path):
if FLAGS.restore_model_path.startswith('s3://'):
restore_model_name = FLAGS.restore_model_path.rsplit('/', 1)[1]
file.copy(FLAGS.restore_model_path,
'/cache/tmp/' + restore_model_name)
model.load_weights('/cache/tmp/' + restore_model_name)
os.remove('/cache/tmp/' + restore_model_name)
else:
model.load_weights(FLAGS.restore_model_path)
if not os.path.exists(FLAGS.train_local):
os.makedirs(FLAGS.train_local)
tensorBoard = TensorBoard(log_dir=FLAGS.train_local)
# reduce_lr = ks.callbacks.ReduceLROnPlateau(monitor='val_acc', factor=0.5, verbose=1, patience=1,
# min_lr=1e-7)
# 余弦退火学习率
sample_count = len(train_sequence) * FLAGS.batch_size
epochs = FLAGS.max_epochs
warmup_epoch = 5
batch_size = FLAGS.batch_size
learning_rate_base = FLAGS.learning_rate
total_steps = int(epochs * sample_count / batch_size)
warmup_steps = int(warmup_epoch * sample_count / batch_size)
warm_up_lr = WarmUpCosineDecayScheduler(
learning_rate_base=learning_rate_base,
total_steps=total_steps,
warmup_learning_rate=0,
warmup_steps=warmup_steps,
hold_base_rate_steps=0,
)
history = LossHistory(FLAGS)
model.fit_generator(train_sequence,
steps_per_epoch=len(train_sequence),
epochs=FLAGS.max_epochs,
verbose=1,
callbacks=[history, tensorBoard, warm_up_lr],
validation_data=validation_sequence,
max_queue_size=10,
workers=int(multiprocessing.cpu_count() * 0.7),
use_multiprocessing=True,
shuffle=True)
print('training done!')
if FLAGS.deploy_script_path != '':
from save_model import save_pb_model
save_pb_model(FLAGS, model)
if FLAGS.test_data_url != '':
print('test dataset predicting...')
from eval import load_test_data
img_names, test_data, test_labels = load_test_data(FLAGS)
predictions = model.predict(test_data, verbose=0)
right_count = 0
for index, pred in enumerate(predictions):
predict_label = np.argmax(pred, axis=0)
test_label = test_labels[index]
if predict_label == test_label:
right_count += 1
accuracy = right_count / len(img_names)
print('accuracy: %0.4f' % accuracy)
metric_file_name = os.path.join(FLAGS.train_local, 'metric.json')
metric_file_content = '{"total_metric": {"total_metric_values": {"accuracy": %0.4f}}}' % accuracy
with open(metric_file_name, "w") as f:
f.write(metric_file_content + '\n')
print('end')
