def test_model(FLAGS, model, need_save=True):
if FLAGS.test_data_url != '':
print('test dataset predicting...')
from eval import load_test_data
img_names, test_data, test_labels = load_test_data(FLAGS)
predictions = model.predict(test_data, verbose=0)
right_count = 0
error_infos = []
for index, pred in enumerate(predictions):
predict_label = np.argmax(pred, axis=0)
test_label = test_labels[index]
if predict_label == test_label:
right_count += 1
else:
error_infos.append(
'%s, %s, %s\n' %
(img_names[index], test_label, predict_label))
accuracy = right_count / len(img_names)
print('accuracy: %0.4f' % accuracy)
if need_save:
result_file_name = os.path.join(FLAGS.train_url, 'accuracy.txt')
with open(result_file_name, 'w') as f:
f.write('# predict error files\n')
f.write('####################################\n')
f.write('file_name, true_label, pred_label\n')
f.writelines(error_infos)
f.write('####################################\n')
f.write('accuracy: %s\n' % accuracy)
def train_model(FLAGS):
# data flow generator
train_sequence, validation_sequence = data_flow(FLAGS.data_local, FLAGS.batch_size,
FLAGS.num_classes, FLAGS.input_size)
optimizer = adam(lr=FLAGS.learning_rate)#, clipnorm=0.001)
reduce_on_plateau = ReduceLROnPlateau(monitor="val_acc", mode="max", factor=0.1, patience=10, verbose=1)
objective = 'categorical_crossentropy'
metrics = ['accuracy']
model = model_fn(FLAGS, objective, optimizer, metrics)
# if FLAGS.restore_model_path != '' and file.exists(FLAGS.restore_model_path):
# if FLAGS.restore_model_path.startswith('s3://'):
# restore_model_name = FLAGS.restore_model_path.rsplit('/', 1)[1]
# file.copy(FLAGS.restore_model_path, '/cache/tmp/' + restore_model_name)
# model.load_weights('/cache/tmp/' + restore_model_name)
# os.remove('/cache/tmp/' + restore_model_name)
# else:
# model.load_weights(FLAGS.restore_model_path)
if not os.path.exists(FLAGS.train_local):
os.makedirs(FLAGS.train_local)
tensorBoard = TensorBoard(log_dir=FLAGS.train_local)
history = LossHistory(FLAGS)
model.fit_generator(
train_sequence,
steps_per_epoch=len(train_sequence),
epochs=FLAGS.max_epochs,
verbose=2,
callbacks=[history, tensorBoard,reduce_on_plateau],
validation_data=validation_sequence,
max_queue_size=10,
workers=int(multiprocessing.cpu_count() * 0.7),
use_multiprocessing=True,
shuffle=True
)
print('training done!')
if FLAGS.deploy_script_path != '':
from save_model import save_pb_model
save_pb_model(FLAGS, model)
if FLAGS.test_data_url != '':
print('test dataset predicting...')
from eval import load_test_data
img_names, test_data, test_labels = load_test_data(FLAGS)
predictions = model.predict(test_data, verbose=0)
right_count = 0
for index, pred in enumerate(predictions):
predict_label = np.argmax(pred, axis=0)
test_label = test_labels[index]
if predict_label == test_label:
right_count += 1
accuracy = right_count / len(img_names)
print('accuracy: %0.4f' % accuracy)
metric_file_name = os.path.join(FLAGS.train_local, 'metric.json')
metric_file_content = '{"total_metric": {"total_metric_values": {"accuracy": %0.4f}}}' % accuracy
with open(metric_file_name, "w") as f:
f.write(metric_file_content + '\n')
print('end')
def train_model(FLAGS):
# data flow generator
train_sequence, validation_sequence = data_flow(FLAGS.data_local,
FLAGS.batch_size,
FLAGS.num_classes,
FLAGS.input_size)
optimizer = adam(lr=FLAGS.learning_rate, clipnorm=0.001)
objective = 'binary_crossentropy'
metrics = ['accuracy']
model = model_fn(FLAGS, objective, optimizer, metrics)
if FLAGS.restore_model_path != '' and file.exists(
FLAGS.restore_model_path):
if FLAGS.restore_model_path.startswith('s3://'):
restore_model_name = FLAGS.restore_model_path.rsplit('/', 1)[1]
file.copy(FLAGS.restore_model_path,
'/cache/tmp/' + restore_model_name)
model.load_weights('/cache/tmp/' + restore_model_name)
os.remove('/cache/tmp/' + restore_model_name)
else:
model.load_weights(FLAGS.restore_model_path)
if not os.path.exists(FLAGS.train_local):
os.makedirs(FLAGS.train_local)
tensorBoard = TensorBoard(log_dir=FLAGS.train_local)
history = LossHistory(FLAGS)
# STEP_SIZE_TRAIN = train_sequence.n
# STEP_SIZE_VALID = validation_sequence.n
model.fit_generator(
train_sequence,
steps_per_epoch=len(train_sequence),
epochs=FLAGS.max_epochs,
verbose=1,
callbacks=[history, tensorBoard],
# validation_steps=STEP_SIZE_VALID,
validation_data=validation_sequence,
max_queue_size=10,
workers=int(multiprocessing.cpu_count() * 0.7),
use_multiprocessing=True,
shuffle=True)
# count=train_sequence.get_count()
# for n in range(FLAGS.max_epochs):
# for i in range(count):
# batch_train_x,batch_train_y = train_sequence.next_batch()
# batch_val_x, batch_val_y=validation_sequence.next_batch()
# model.fit(x=batch_train_x,
# y=batch_train_y,
# verbose=1,
# callbacks=[history, tensorBoard],
# # validation_steps=STEP_SIZE_VALID,
# validation_data=(batch_val_x, batch_val_y),
# shuffle=True)
print('training done!')
if FLAGS.deploy_script_path != '':
from save_model import save_pb_model
save_pb_model(FLAGS, model)
if FLAGS.test_data_url != '':
print('test dataset predicting...')
from eval import load_test_data
img_names, test_data, test_labels = load_test_data(FLAGS)
predictions = model.predict(test_data, verbose=0)
right_count = 0
for index, pred in enumerate(predictions):
predict_label = np.argmax(pred, axis=0)
test_label = test_labels[index]
if predict_label == test_label:
right_count += 1
accuracy = right_count / len(img_names)
print('accuracy: %0.4f' % accuracy)
metric_file_name = os.path.join(FLAGS.train_local, 'metric.json')
metric_file_content = '{"total_metric": {"total_metric_values": {"accuracy": %0.4f}}}' % accuracy
with open(metric_file_name, "w") as f:
f.write(metric_file_content + '\n')
print('end')
def train_model(FLAGS):
preprocess_input = efn.preprocess_input
train_sequence, validation_sequence = data_flow(FLAGS.data_local,
FLAGS.batch_size,
FLAGS.num_classes,
FLAGS.input_size,
preprocess_input)
optimizer = Adam(lr=FLAGS.learning_rate)
objective = 'categorical_crossentropy'
metrics = ['accuracy']
model = model_fn(FLAGS, objective, optimizer, metrics)
if FLAGS.restore_model_path != '' and os.path.exists(
FLAGS.restore_model_path):
model.load_weights(FLAGS.restore_model_path)
print("LOAD OK!!!")
if not os.path.exists(FLAGS.save_model_local):
os.makedirs(FLAGS.save_model_local)
log_local = './log_file/'
tensorBoard = TensorBoard(log_dir=log_local)
# reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5, mode='auto')
sample_count = len(train_sequence) * FLAGS.batch_size
epochs = FLAGS.max_epochs
warmup_epoch = 5
batch_size = FLAGS.batch_size
learning_rate_base = FLAGS.learning_rate
total_steps = int(epochs * sample_count / batch_size)
warmup_steps = int(warmup_epoch * sample_count / batch_size)
warm_up_lr = WarmUpCosineDecayScheduler(
learning_rate_base=learning_rate_base,
total_steps=total_steps,
warmup_learning_rate=0,
warmup_steps=warmup_steps,
hold_base_rate_steps=0,
)
cbk = Mycbk(FLAGS)
model.fit_generator(train_sequence,
steps_per_epoch=len(train_sequence),
epochs=FLAGS.max_epochs,
verbose=1,
callbacks=[cbk, tensorBoard, warm_up_lr],
validation_data=validation_sequence,
max_queue_size=10,
workers=int(multiprocessing.cpu_count() * 0.7),
use_multiprocessing=True,
shuffle=True)
print('training done!')
from save_model import save_pb_model
save_pb_model(FLAGS, model)
if FLAGS.test_data_local != '':
print('test dataset predicting...')
from eval import load_test_data
img_names, test_data, test_labels = load_test_data(FLAGS)
test_data = preprocess_input(test_data)
predictions = model.predict(test_data, verbose=0)
right_count = 0
for index, pred in enumerate(predictions):
predict_label = np.argmax(pred, axis=0)
test_label = test_labels[index]
if predict_label == test_label:
right_count += 1
accuracy = right_count / len(img_names)
print('accuracy: %0.4f' % accuracy)
metric_file_name = os.path.join(FLAGS.save_model_local, 'metric.json')
metric_file_content = '{"total_metric": {"total_metric_values": {"accuracy": %0.4f}}}' % accuracy
with open(metric_file_name, "w") as f:
f.write(metric_file_content + '\n')
print('end')
parse.add_argument('--reg_coeff', type=float, default=0.0001)
parse.add_argument('--momentum', type=float, default=0.8)
parse.add_argument('--adam_beta1', type=float, default=0.9)
parse.add_argument('--adam_beta2', type=float, default=0.999)
parse.add_argument('--adam_epsilon', type=float, default=1e-8)
parse.add_argument('--np_seed', type=int, default=123)
parse.add_argument('--tf_seed', type=int, default=1234)
parse.add_argument('--restart', action='store_true')
parse.add_argument('--no_summary', action='store_true')
parse.add_argument('--summary_prefix', default='')
return parse
if __name__ == '__main__':
# arguments
parse = build_argparser()
args = parse.parse_args()
# model
model = importlib.import_module('models.%s' % args.model)
# load data
x, y = load_train_data()
if args.test_model:
tx, ty = load_test_data()
# train model
train(x, y, vars(args), model.build_model, tx, ty)
else:
train(x, y, vars(args), model.build_model)
def train_model(FLAGS):
# data flow generator
train_sequence, validation_sequence = data_flow(FLAGS.data_local,
FLAGS.batch_size,
FLAGS.num_classes,
FLAGS.input_size)
# optimizer = adam(lr=FLAGS.learning_rate, clipnorm=0.001)
optimizer = Nadam(lr=FLAGS.learning_rate,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
schedule_decay=0.004)
# optimizer = SGD(lr=FLAGS.learning_rate, momentum=0.9)
objective = 'categorical_crossentropy'
metrics = ['accuracy']
#model = model_fn(FLAGS, objective, optimizer, metrics)
#model = model_fn_SE_ResNet50(FLAGS, objective, optimizer, metrics)
model = model_fn_Xception(FLAGS, objective, optimizer, metrics)
if FLAGS.restore_model_path != '' and os.path.exists(
FLAGS.restore_model_path):
if FLAGS.restore_model_path.startswith('s3://'):
restore_model_name = FLAGS.restore_model_path.rsplit('/', 1)[1]
shutil.copyfile(FLAGS.restore_model_path,
'/cache/tmp/' + restore_model_name)
model.load_weights('/cache/tmp/' + restore_model_name)
os.remove('/cache/tmp/' + restore_model_name)
else:
model.load_weights(FLAGS.restore_model_path)
print("LOAD OK!!!")
if not os.path.exists(FLAGS.train_local):
os.makedirs(FLAGS.train_local)
log_local = '../log_file/'
tensorBoard = TensorBoard(log_dir=log_local)
# reduce_lr = ks.callbacks.ReduceLROnPlateau(monitor='val_acc', factor=0.5, verbose=1, patience=1,
# min_lr=1e-7)
# 余弦退火学习率
sample_count = len(train_sequence) * FLAGS.batch_size
epochs = FLAGS.max_epochs
warmup_epoch = 5
batch_size = FLAGS.batch_size
learning_rate_base = FLAGS.learning_rate
total_steps = int(epochs * sample_count / batch_size)
warmup_steps = int(warmup_epoch * sample_count / batch_size)
warm_up_lr = WarmUpCosineDecayScheduler(
learning_rate_base=learning_rate_base,
total_steps=total_steps,
warmup_learning_rate=0,
warmup_steps=warmup_steps,
hold_base_rate_steps=0,
)
history = LossHistory(FLAGS)
model.fit_generator(train_sequence,
steps_per_epoch=len(train_sequence),
epochs=FLAGS.max_epochs,
verbose=1,
callbacks=[history, tensorBoard, warm_up_lr],
validation_data=validation_sequence,
max_queue_size=10,
workers=int(multiprocessing.cpu_count() * 0.7),
use_multiprocessing=True,
shuffle=True)
print('training done!')
if FLAGS.deploy_script_path != '':
from save_model import save_pb_model
save_pb_model(FLAGS, model)
if FLAGS.test_data_url != '':
print('test dataset predicting...')
from eval import load_test_data
img_names, test_data, test_labels = load_test_data(FLAGS)
test_data = preprocess_input(test_data)
predictions = model.predict(test_data, verbose=0)
right_count = 0
for index, pred in enumerate(predictions):
predict_label = np.argmax(pred, axis=0)
test_label = test_labels[index]
if predict_label == test_label:
right_count += 1
accuracy = right_count / len(img_names)
print('accuracy: %0.4f' % accuracy)
metric_file_name = os.path.join(FLAGS.train_local, 'metric.json')
metric_file_content = '{"total_metric": {"total_metric_values": {"accuracy": %0.4f}}}' % accuracy
with open(metric_file_name, "w") as f:
f.write(metric_file_content + '\n')
print('end')