def main():
parser = argparse.ArgumentParser(
description='Pick out VOC object from COCO annotation dataset')
parser.add_argument('--coco_annotation_file',
type=str,
required=True,
help='coco annotation txt file')
parser.add_argument(
'--coco_classes_path',
type=str,
default='../../configs/coco_classes.txt',
help=
'path to coco class definitions, default ../../configs/coco_classes.txt'
)
parser.add_argument(
'--voc_classes_path',
type=str,
default='../../configs/voc_classes.txt',
help=
'path to voc class definitions, default ../../configs/voc_classes.txt')
parser.add_argument('--output_voc_annotation_file',
type=str,
required=True,
help='output voc classes annotation file')
args = parser.parse_args()
# param parse
coco_class_names = get_classes(args.coco_classes_path)
voc_class_names = get_classes(args.voc_classes_path)
coco_annotation_lines = get_dataset(args.coco_annotation_file)
output_file = open(args.output_voc_annotation_file, 'w')
for coco_annotation_line in coco_annotation_lines:
# parse annotation line
coco_line = coco_annotation_line.split()
image_name = coco_line[0]
boxes = np.array([
np.array(list(map(int, box.split(',')))) for box in coco_line[1:]
])
has_voc_object = False
for box in boxes:
coco_class_id = box[-1]
# check if coco object in voc class list
# if true, keep the image & box info
if coco_class_names[coco_class_id] in voc_class_names:
if has_voc_object == False:
has_voc_object = True
output_file.write(image_name)
# get VOC class ID of the COCO object
voc_class_id = voc_class_names.index(
coco_class_names[coco_class_id])
output_file.write(" " + ",".join([str(b) for b in box[:-2]]) +
',' + str(voc_class_id))
if has_voc_object == True:
output_file.write('\n')
output_file.close()
def main():
# class YOLO defines the default value, so suppress any default here
parser = argparse.ArgumentParser(argument_default=argparse.SUPPRESS, description='evaluate YOLO model (h5/pb/onnx/tflite/mnn) with test dataset')
'''
Command line options
'''
parser.add_argument(
'--model_path', type=str, required=True,
help='path to model file')
parser.add_argument(
'--anchors_path', type=str, required=True,
help='path to anchor definitions')
parser.add_argument(
'--classes_path', type=str, required=False,
help='path to class definitions, default configs/voc_classes.txt', default=os.path.join('configs' , 'voc_classes.txt'))
parser.add_argument(
'--annotation_file', type=str, required=True,
help='test annotation txt file')
parser.add_argument(
'--eval_type', type=str,
help='evaluation type (VOC/COCO), default=VOC', default='VOC')
parser.add_argument(
'--iou_threshold', type=float,
help='IOU threshold for PascalVOC mAP, default=0.5', default=0.5)
parser.add_argument(
'--conf_threshold', type=float,
help='confidence threshold for filtering box in postprocess, default=0.001', default=0.001)
parser.add_argument(
'--model_image_size', type=str,
help='model image input size as <height>x<width>, default 416x416', default='416x416')
parser.add_argument(
'--save_result', default=False, action="store_true",
help='Save the detection result image in result/detection dir'
)
args = parser.parse_args()
# param parse
anchors = get_anchors(args.anchors_path)
class_names = get_classes(args.classes_path)
height, width = args.model_image_size.split('x')
model_image_size = (int(height), int(width))
assert (model_image_size[0]%32 == 0 and model_image_size[1]%32 == 0), 'model_image_size should be multiples of 32'
annotation_lines = get_dataset(args.annotation_file, shuffle=False)
model, model_format = load_eval_model(args.model_path)
start = time.time()
eval_AP(model, model_format, annotation_lines, anchors, class_names, model_image_size, args.eval_type, args.iou_threshold, args.conf_threshold, args.save_result)
end = time.time()
print("Evaluation time cost: {:.6f}s".format(end - start))
def main():
# class YOLO defines the default value, so suppress any default here
parser = argparse.ArgumentParser(argument_default=argparse.SUPPRESS, description='evaluate YOLO model (h5/pb/tflite/mnn) with test dataset')
'''
Command line options
'''
parser.add_argument(
'--model_path', type=str, required=True,
help='path to model file')
parser.add_argument(
'--custom_objects', type=str, required=False, default=None,
help="Custom objects in keras model (swish/tf). Separated with comma if more than one.")
parser.add_argument(
'--anchors_path', type=str, required=True,
help='path to anchor definitions')
parser.add_argument(
'--classes_path', type=str, required=False,
help='path to class definitions, default configs/voc_classes.txt', default='configs/voc_classes.txt')
parser.add_argument(
'--annotation_file', type=str, required=True,
help='test annotation txt file')
parser.add_argument(
'--eval_type', type=str,
help='evaluation type (VOC/COCO), default=VOC', default='VOC')
parser.add_argument(
'--iou_threshold', type=float,
help='IOU threshold for PascalVOC mAP, default=0.5', default=0.5)
parser.add_argument(
'--conf_threshold', type=float,
help='confidence threshold for filtering box in postprocess, default=0.001', default=0.001)
parser.add_argument(
'--model_image_size', type=str,
help='model image input size as <num>x<num>, default 416x416', default='416x416')
parser.add_argument(
'--save_result', default=False, action="store_true",
help='Save the detection result image in result/detection dir'
)
args = parser.parse_args()
# param parse
anchors = get_anchors(args.anchors_path)
class_names = get_classes(args.classes_path)
height, width = args.model_image_size.split('x')
model_image_size = (int(height), int(width))
annotation_lines = get_dataset(args.annotation_file)
model, model_format = load_eval_model(args.model_path, args.custom_objects)
eval_AP(model, model_format, annotation_lines, anchors, class_names, model_image_size, args.eval_type, args.iou_threshold, args.conf_threshold, args.save_result)
def main():
parser = argparse.ArgumentParser(
argument_default=argparse.SUPPRESS,
description='Test tool for mosaic data augment function')
parser.add_argument('--annotation_file',
type=str,
required=True,
help='data annotation txt file')
parser.add_argument('--classes_path',
type=str,
required=True,
help='path to class definitions')
parser.add_argument(
'--output_path',
type=str,
required=False,
help='output path for augmented images, default is ./test',
default='./test')
parser.add_argument('--batch_size',
type=int,
required=False,
help="batch size for test data, default=16",
default=16)
parser.add_argument(
'--model_image_size',
type=str,
required=False,
help='model image input size as <num>x<num>, default 416x416',
default='416x416')
args = parser.parse_args()
class_names = get_classes(args.classes_path)
height, width = args.model_image_size.split('x')
model_image_size = (int(height), int(width))
annotation_lines = get_dataset(args.annotation_file)
os.makedirs(args.output_path, exist_ok=True)
image_data = []
boxes_data = []
for i in range(args.batch_size):
annotation_line = annotation_lines[i]
image, boxes = get_ground_truth_data(annotation_line,
input_shape=model_image_size,
augment=True)
#un-normalize image
image = image * 255.0
image = image.astype(np.uint8)
image_data.append(image)
boxes_data.append(boxes)
image_data = np.array(image_data)
boxes_data = np.array(boxes_data)
image_data, boxes_data = random_mosaic_augment(image_data,
boxes_data,
jitter=1)
draw_boxes(image_data, boxes_data, class_names, args.output_path)
def dataset_visualize(annotation_file, classes_path):
annotation_lines = get_dataset(annotation_file, shuffle=False)
# get class names and count class item number
class_names = get_classes(classes_path)
colors = get_colors(len(class_names))
pbar = tqdm(total=len(annotation_lines), desc='Visualize dataset')
for i, annotation_line in enumerate(annotation_lines):
pbar.update(1)
line = annotation_line.split()
image = Image.open(line[0]).convert('RGB')
image = np.array(image, dtype='uint8')
boxes = np.array(
[np.array(list(map(int, box.split(',')))) for box in line[1:]])
classes = boxes[:, -1]
boxes = boxes[:, :-1]
scores = np.array([1.0] * len(classes))
image = draw_boxes(image,
boxes,
classes,
scores,
class_names,
colors,
show_score=False)
# show image file info
image_file_name = os.path.basename(line[0])
cv2.putText(image,
image_file_name +
'({}/{})'.format(i + 1, len(annotation_lines)), (3, 15),
cv2.FONT_HERSHEY_PLAIN,
fontScale=1,
color=(255, 0, 0),
thickness=1,
lineType=cv2.LINE_AA)
# convert to BGR for cv2.imshow
image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
cv2.namedWindow("Image", 0)
cv2.imshow("Image", image)
keycode = cv2.waitKey(0) & 0xFF
if keycode == ord('q') or keycode == 27: # 27 is keycode for Esc
break
pbar.close()
def main():
parser = argparse.ArgumentParser(argument_default=argparse.SUPPRESS, description='Test tool for enhance mosaic data augment function')
parser.add_argument('--annotation_file', type=str, required=True, help='data annotation txt file')
parser.add_argument('--classes_path', type=str, required=True, help='path to class definitions')
parser.add_argument('--output_path', type=str, required=False, help='output path for augmented images, default is ./test', default='./test')
parser.add_argument('--batch_size', type=int, required=False, help = "batch size for test data, default=16", default=16)
parser.add_argument('--model_image_size', type=str, required=False, help='model image input size as <height>x<width>, default 416x416', default='416x416')
parser.add_argument('--augment_type', type=str, required=False, help = "enhance data augmentation type (mosaic/cutmix), default=mosaic", default='mosaic')
args = parser.parse_args()
class_names = get_classes(args.classes_path)
height, width = args.model_image_size.split('x')
model_image_size = (int(height), int(width))
assert (model_image_size[0]%32 == 0 and model_image_size[1]%32 == 0), 'model_image_size should be multiples of 32'
annotation_lines = get_dataset(args.annotation_file)
os.makedirs(args.output_path, exist_ok=True)
image_data = []
boxes_data = []
for i in range(args.batch_size):
annotation_line = annotation_lines[i]
image, boxes = get_ground_truth_data(annotation_line, input_shape=model_image_size, augment=True)
#un-normalize image
image = image*255.0
image = image.astype(np.uint8)
image_data.append(image)
boxes_data.append(boxes)
image_data = np.array(image_data)
boxes_data = np.array(boxes_data)
if args.augment_type == 'mosaic':
image_data, boxes_data = random_mosaic_augment(image_data, boxes_data, prob=1)
elif args.augment_type == 'cutmix':
image_data, boxes_data = random_cutmix_augment(image_data, boxes_data, prob=1)
else:
raise ValueError('Unsupported augment type')
draw_boxes(image_data, boxes_data, class_names, args.output_path)
def main(args):
annotation_file = args.annotation_file
log_dir = os.path.join('logs', '000')
classes_path = args.classes_path
class_names = get_classes(classes_path)
num_classes = len(class_names)
print('classes_path =', classes_path)
print('class_names = ', class_names)
print('num_classes = ', num_classes)
anchors = get_anchors(args.anchors_path)
num_anchors = len(anchors)
# get freeze level according to CLI option
if args.weights_path:
freeze_level = 0
else:
freeze_level = 1
if args.freeze_level is not None:
freeze_level = args.freeze_level
# callbacks for training process
logging = TensorBoard(log_dir=log_dir,
histogram_freq=0,
write_graph=False,
write_grads=False,
write_images=False,
update_freq='batch')
checkpoint = ModelCheckpoint(os.path.join(
log_dir, 'ep{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}.h5'),
monitor='val_loss',
verbose=1,
save_weights_only=False,
save_best_only=True,
period=1)
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=10,
verbose=1,
cooldown=0,
min_lr=1e-10)
early_stopping = EarlyStopping(monitor='val_loss',
min_delta=0,
patience=50,
verbose=1)
terminate_on_nan = TerminateOnNaN()
callbacks = [
logging, checkpoint, reduce_lr, early_stopping, terminate_on_nan
]
# get train&val dataset
dataset = get_dataset(annotation_file)
if args.val_annotation_file:
val_dataset = get_dataset(args.val_annotation_file)
num_train = len(dataset)
print('num_train = ', num_train)
num_val = len(val_dataset)
dataset.extend(val_dataset)
else:
val_split = args.val_split
num_val = int(len(dataset) * val_split)
num_train = len(dataset) - num_val
# assign multiscale interval
if args.multiscale:
rescale_interval = args.rescale_interval
else:
rescale_interval = -1 #Doesn't rescale
# model input shape check
input_shape = args.model_image_size
assert (input_shape[0] % 32 == 0
and input_shape[1] % 32 == 0), 'Multiples of 32 required'
# get different model type & train&val data generator
if num_anchors == 9:
# YOLOv3 use 9 anchors
get_train_model = get_yolo3_train_model
data_generator = yolo3_data_generator_wrapper
# tf.keras.Sequence style data generator
#train_data_generator = Yolo3DataGenerator(dataset[:num_train], args.batch_size, input_shape, anchors, num_classes, args.enhance_augment, rescale_interval)
#val_data_generator = Yolo3DataGenerator(dataset[num_train:], args.batch_size, input_shape, anchors, num_classes)
tiny_version = False
elif num_anchors == 6:
# Tiny YOLOv3 use 6 anchors
get_train_model = get_yolo3_train_model
data_generator = yolo3_data_generator_wrapper
# tf.keras.Sequence style data generator
#train_data_generator = Yolo3DataGenerator(dataset[:num_train], args.batch_size, input_shape, anchors, num_classes, args.enhance_augment, rescale_interval)
#val_data_generator = Yolo3DataGenerator(dataset[num_train:], args.batch_size, input_shape, anchors, num_classes)
tiny_version = True
elif num_anchors == 5:
# YOLOv2 use 5 anchors
get_train_model = get_yolo2_train_model
data_generator = yolo2_data_generator_wrapper
# tf.keras.Sequence style data generator
#train_data_generator = Yolo2DataGenerator(dataset[:num_train], args.batch_size, input_shape, anchors, num_classes, args.enhance_augment, rescale_interval)
#val_data_generator = Yolo2DataGenerator(dataset[num_train:], args.batch_size, input_shape, anchors, num_classes)
tiny_version = False
else:
raise ValueError('Unsupported anchors number')
# prepare online evaluation callback
if args.eval_online:
eval_callback = EvalCallBack(
args.model_type,
dataset[num_train:],
anchors,
class_names,
args.model_image_size,
args.model_pruning,
log_dir,
eval_epoch_interval=args.eval_epoch_interval,
save_eval_checkpoint=args.save_eval_checkpoint)
callbacks.append(eval_callback)
# prepare train/val data shuffle callback
if args.data_shuffle:
shuffle_callback = DatasetShuffleCallBack(dataset)
callbacks.append(shuffle_callback)
# prepare model pruning config
pruning_end_step = np.ceil(1.0 * num_train / args.batch_size).astype(
np.int32) * args.total_epoch
if args.model_pruning:
pruning_callbacks = [
sparsity.UpdatePruningStep(),
sparsity.PruningSummaries(log_dir=log_dir, profile_batch=0)
]
callbacks = callbacks + pruning_callbacks
# prepare optimizer
optimizer = get_optimizer(args.optimizer,
args.learning_rate,
decay_type=None)
# get train model
model = get_train_model(args.model_type,
anchors,
num_classes,
weights_path=args.weights_path,
freeze_level=freeze_level,
optimizer=optimizer,
label_smoothing=args.label_smoothing,
model_pruning=args.model_pruning,
pruning_end_step=pruning_end_step)
# support multi-gpu training
template_model = None
if args.gpu_num >= 2:
# keep the template model for saving result
template_model = model
model = multi_gpu_model(model, gpus=args.gpu_num)
# recompile multi gpu model
model.compile(optimizer=optimizer,
loss={
'yolo_loss': lambda y_true, y_pred: y_pred
})
model.summary()
# Transfer training some epochs with frozen layers first if needed, to get a stable loss.
initial_epoch = args.init_epoch #####################################################################################################
epochs = initial_epoch + args.transfer_epoch
print("Transfer training stage")
print(
'Train on {} samples, val on {} samples, with batch size {}, input_shape {}.'
.format(num_train, num_val, args.batch_size, input_shape))
#model.fit_generator(train_data_generator,
model.fit_generator(
data_generator(dataset[:num_train], args.batch_size, input_shape,
anchors, num_classes, args.enhance_augment),
steps_per_epoch=max(1, num_train // args.batch_size),
#validation_data=val_data_generator,
validation_data=data_generator(dataset[num_train:], args.batch_size,
input_shape, anchors, num_classes),
validation_steps=max(1, num_val // args.batch_size),
epochs=epochs,
initial_epoch=initial_epoch,
#verbose=1,
workers=1,
use_multiprocessing=False,
max_queue_size=10,
callbacks=callbacks)
# Wait 2 seconds for next stage
time.sleep(2)
if args.decay_type:
# rebuild optimizer to apply learning rate decay, only after
# unfreeze all layers
callbacks.remove(reduce_lr)
steps_per_epoch = max(1, num_train // args.batch_size)
decay_steps = steps_per_epoch * (args.total_epoch - args.init_epoch -
args.transfer_epoch)
optimizer = get_optimizer(args.optimizer,
args.learning_rate,
decay_type=args.decay_type,
decay_steps=decay_steps)
# Unfreeze the whole network for further tuning
# NOTE: more GPU memory is required after unfreezing the body
print("Unfreeze and continue training, to fine-tune.")
for i in range(len(model.layers)):
model.layers[i].trainable = True
model.compile(optimizer=optimizer,
loss={
'yolo_loss': lambda y_true, y_pred: y_pred
}) # recompile to apply the change
print(
'Train on {} samples, val on {} samples, with batch size {}, input_shape {}.'
.format(num_train, num_val, args.batch_size, input_shape))
#model.fit_generator(train_data_generator,
model.fit_generator(
data_generator(dataset[:num_train], args.batch_size, input_shape,
anchors, num_classes, args.enhance_augment,
rescale_interval),
steps_per_epoch=max(1, num_train // args.batch_size),
#validation_data=val_data_generator,
validation_data=data_generator(dataset[num_train:], args.batch_size,
input_shape, anchors, num_classes),
validation_steps=max(1, num_val // args.batch_size),
epochs=args.total_epoch,
initial_epoch=epochs,
#verbose=1,
workers=1,
use_multiprocessing=False,
max_queue_size=10,
callbacks=callbacks)
# Finally store model
if args.model_pruning:
if template_model is not None:
template_model = sparsity.strip_pruning(template_model)
else:
model = sparsity.strip_pruning(model)
if template_model is not None:
template_model.save(os.path.join(log_dir, 'trained_final.h5'))
else:
model.save(os.path.join(log_dir, 'trained_final.h5'))
def get_evaluation_metrics(
model,
tokenizer,
data_dir: str,
filename: str,
per_gpu_batch_size: int = 32,
num_batches: int = None,
disable_tqdm: bool = False,
) -> Dict[str, "Number"]:
"""
Return an OrderedDict in the format:
{
'exact': 0.8169797018445212,
'f1': 4.4469722448269335,
'total': 11873,
'HasAns_exact': 0.15182186234817813,
'HasAns_f1': 7.422216845956518,
'HasAns_total': 5928,
'NoAns_exact': 1.4802354920100924,
'NoAns_f1': 1.4802354920100924,
'NoAns_total': 5945,
'best_exact': 50.07159100480081,
'best_exact_thresh': 0.0,
'best_f1': 50.0772059855695,
'best_f1_thresh': 0.0
}
"""
# These are not used in inference, only for scoring in `compute_predictions_logits()`.
processor = SquadV2Processor()
examples: List[SquadExample] = processor.get_dev_examples(
data_dir, filename=filename)
features: List[SquadFeatures] = get_dataset(
tokenizer=tokenizer,
processor=processor,
data_dir=data_dir,
filename=filename,
per_gpu_batch_size=per_gpu_batch_size,
shard=False,
shuffle=False,
drop_remainder=False,
return_raw_features=True,
)
# Here we get the dataset instead of just the features, with return_raw_features=False.
dataset: tf.data.Dataset = get_dataset(
tokenizer=tokenizer,
processor=processor,
data_dir=data_dir,
filename=filename,
per_gpu_batch_size=per_gpu_batch_size,
shard=False,
shuffle=False,
drop_remainder=False,
return_raw_features=False,
)
results: List[SquadResult] = get_squad_results(
model=model,
dataset=dataset,
features=features,
per_gpu_batch_size=per_gpu_batch_size,
num_batches=num_batches,
disable_tqdm=disable_tqdm,
)
write_prediction_files = False
if write_prediction_files:
output_predictions_file = f"/fsx/{args.checkpoint}_predictions.json"
output_nbest_file = f"/fsx/{args.checkpoint}_nbest_predictions.json"
output_null_log_odds_file = f"/fsx/{args.checkpoint}_null_odds.json"
else:
output_predictions_file = None
output_nbest_file = None
output_null_log_odds_file = None
predictions = compute_predictions_logits(
all_examples=examples,
all_features=features,
all_results=results,
n_best_size=20,
max_answer_length=30,
do_lower_case=True,
output_prediction_file=output_predictions_file,
output_nbest_file=output_nbest_file,
output_null_log_odds_file=output_null_log_odds_file,
verbose_logging=False,
version_2_with_negative=True,
null_score_diff_threshold=0.0,
tokenizer=tokenizer,
)
results: collections.OrderedDict = squad_evaluate(examples, predictions)
return results
def main():
parser = argparse.ArgumentParser(
argument_default=argparse.SUPPRESS,
description='Test tool for enhance mosaic data augment function')
parser.add_argument('--annotation_file',
type=str,
required=True,
help='data annotation txt file')
parser.add_argument('--classes_path',
type=str,
required=True,
help='path to class definitions')
parser.add_argument(
'--output_path',
type=str,
required=False,
help='output path for augmented images, default=%(default)s',
default='./test')
parser.add_argument('--batch_size',
type=int,
required=False,
help="batch size for test data, default=%(default)s",
default=16)
parser.add_argument(
'--model_input_shape',
type=str,
required=False,
help='model image input shape as <height>x<width>, default=%(default)s',
default='416x416')
parser.add_argument(
'--enhance_augment',
type=str,
required=False,
help="enhance data augmentation type, default=%(default)s",
default=None,
choices=['mosaic', 'mosaic_v5', 'cutmix', None])
args = parser.parse_args()
class_names = get_classes(args.classes_path)
height, width = args.model_input_shape.split('x')
model_input_shape = (int(height), int(width))
assert (model_input_shape[0] % 32 == 0 and model_input_shape[1] % 32
== 0), 'model_input_shape should be multiples of 32'
annotation_lines = get_dataset(args.annotation_file)
os.makedirs(args.output_path, exist_ok=True)
image_data = []
boxes_data = []
pbar = tqdm(total=args.batch_size, desc='Generate augment image')
for i in range(args.batch_size):
pbar.update(1)
annotation_line = annotation_lines[i]
image, boxes = get_ground_truth_data(annotation_line,
input_shape=model_input_shape,
augment=True)
#denormalize image
image = denormalize_image(image)
image_data.append(image)
boxes_data.append(boxes)
pbar.close()
image_data = np.array(image_data)
boxes_data = np.array(boxes_data)
if args.enhance_augment == 'mosaic':
image_data, boxes_data = random_mosaic_augment(image_data,
boxes_data,
prob=1)
elif args.enhance_augment == 'mosaic_v5':
image_data, boxes_data = random_mosaic_augment_v5(image_data,
boxes_data,
prob=1)
elif args.enhance_augment == 'cutmix':
image_data, boxes_data = random_cutmix_augment(image_data,
boxes_data,
prob=1)
elif args.enhance_augment == None:
print('No enhance augment type. Will only apply base augment')
else:
raise ValueError('Unsupported augment type')
draw_boxes(image_data, boxes_data, class_names, args.output_path)
print('Done. augment images have been saved in', args.output_path)
def run_squad_and_get_results(
model: tf.keras.Model, # Must be QuestionAnswering model, not PreTraining
tokenizer: PreTrainedTokenizer,
run_name: str,
filesystem_prefix: str,
per_gpu_batch_size: int,
checkpoint_frequency: Optional[int],
validate_frequency: Optional[int],
evaluate_frequency: Optional[int],
learning_rate: float,
warmup_steps: int,
total_steps: int,
dataset: str,
dummy_eval: bool = False,
) -> Dict:
checkpoint_frequency = checkpoint_frequency or 1000000
validate_frequency = validate_frequency or 1000000
evaluate_frequency = evaluate_frequency or 1000000
is_sagemaker = filesystem_prefix.startswith("/opt/ml")
disable_tqdm = is_sagemaker
schedule = LinearWarmupPolyDecaySchedule(
max_learning_rate=learning_rate,
end_learning_rate=0,
warmup_steps=warmup_steps,
total_steps=total_steps,
)
optimizer = tfa.optimizers.AdamW(weight_decay=0.0, learning_rate=schedule)
optimizer = tf.train.experimental.enable_mixed_precision_graph_rewrite(
optimizer, loss_scale="dynamic"
) # AMP
if dataset == "squadv1":
train_filename = "train-v1.1.json"
val_filename = "dev-v1.1.json"
processor = SquadV1Processor()
elif dataset == "squadv2":
train_filename = "train-v2.0.json"
val_filename = "dev-v2.0.json"
processor = SquadV2Processor()
elif dataset == "debug":
train_filename = "dev-v2.0.json"
val_filename = "dev-v2.0.json"
processor = SquadV2Processor()
else:
assert False, "--dataset must be one of ['squadv1', 'squadv2', 'debug']"
data_dir = os.path.join(filesystem_prefix, "squad_data")
train_dataset = get_dataset(
tokenizer=tokenizer,
processor=processor,
data_dir=data_dir,
filename=train_filename,
per_gpu_batch_size=per_gpu_batch_size,
shard=True,
shuffle=True,
repeat=True,
drop_remainder=True,
)
if hvd.rank() == 0:
logger.info(f"Starting finetuning on {dataset}")
pbar = tqdm.tqdm(total_steps, disable=disable_tqdm)
summary_writer = None # Only create a writer if we make it through a successful step
val_dataset = get_dataset(
tokenizer=tokenizer,
processor=processor,
data_dir=data_dir,
filename=val_filename,
per_gpu_batch_size=per_gpu_batch_size,
shard=False,
shuffle=True,
drop_remainder=False,
)
# Need to re-wrap every time this function is called
# Wrapping train_step gives an error with optimizer initialization on the second pass
# of run_squad_and_get_results(). Bug report at https://github.com/tensorflow/tensorflow/issues/38875
# Discussion at https://github.com/tensorflow/tensorflow/issues/27120
global train_step
train_step = rewrap_tf_function(train_step)
for step, batch in enumerate(train_dataset):
learning_rate = schedule(step=tf.constant(step, dtype=tf.float32))
loss, acc, exact_match, f1, precision, recall = train_step(
model=model, optimizer=optimizer, batch=batch
)
# Broadcast model after the first step so parameters and optimizer are initialized
if step == 0:
hvd.broadcast_variables(model.variables, root_rank=0)
hvd.broadcast_variables(optimizer.variables(), root_rank=0)
is_final_step = step >= total_steps - 1
if hvd.rank() == 0:
do_checkpoint = ((step > 0) and step % checkpoint_frequency == 0) or is_final_step
do_validate = ((step > 0) and step % validate_frequency == 0) or is_final_step
do_evaluate = ((step > 0) and step % evaluate_frequency == 0) or is_final_step
pbar.update(1)
description = f"Loss: {loss:.3f}, Acc: {acc:.3f}, EM: {exact_match:.3f}, F1: {f1:.3f}"
pbar.set_description(description)
if do_validate:
logger.info("Running validation")
(
val_loss,
val_acc,
val_exact_match,
val_f1,
val_precision,
val_recall,
) = run_validation(model=model, val_dataset=val_dataset)
description = (
f"Step {step} validation - Loss: {val_loss:.3f}, Acc: {val_acc:.3f}, "
f"EM: {val_exact_match:.3f}, F1: {val_f1:.3f}"
)
logger.info(description)
if do_evaluate:
logger.info("Running evaluation")
if dummy_eval:
results = {
"exact": 0.8169797018445212,
"f1": 4.4469722448269335,
"total": 11873,
"HasAns_exact": 0.15182186234817813,
"HasAns_f1": 7.422216845956518,
"HasAns_total": 5928,
"NoAns_exact": 1.4802354920100924,
"NoAns_f1": 1.4802354920100924,
"NoAns_total": 5945,
"best_exact": 50.07159100480081,
"best_exact_thresh": 0.0,
"best_f1": 50.0772059855695,
"best_f1_thresh": 0.0,
}
else:
results: Dict = get_evaluation_metrics(
model=model,
tokenizer=tokenizer,
data_dir=data_dir,
filename=val_filename,
per_gpu_batch_size=32,
)
print_eval_metrics(results=results, step=step, dataset=dataset)
if do_checkpoint:
# TODO: Abstract out to specify any checkpoint path
checkpoint_path = os.path.join(
filesystem_prefix, f"checkpoints/squad/{run_name}-step{step}.ckpt"
)
logger.info(f"Saving checkpoint at {checkpoint_path}")
model.save_weights(checkpoint_path)
if summary_writer is None:
# TODO: Abstract out to specify any logs path
summary_writer = tf.summary.create_file_writer(
os.path.join(filesystem_prefix, f"logs/squad/{run_name}")
)
with summary_writer.as_default():
tf.summary.scalar("learning_rate", learning_rate, step=step)
tf.summary.scalar("train_loss", loss, step=step)
tf.summary.scalar("train_acc", acc, step=step)
tf.summary.scalar("train_exact", exact_match, step=step)
tf.summary.scalar("train_f1", f1, step=step)
tf.summary.scalar("train_precision", precision, step=step)
tf.summary.scalar("train_recall", recall, step=step)
if do_validate:
tf.summary.scalar("val_loss", val_loss, step=step)
tf.summary.scalar("val_acc", val_acc, step=step)
tf.summary.scalar("val_exact", val_exact_match, step=step)
tf.summary.scalar("val_f1", val_f1, step=step)
tf.summary.scalar("val_precision", val_precision, step=step)
tf.summary.scalar("val_recall", val_recall, step=step)
# And the eval metrics
tensorboard_eval_metrics(
summary_writer=summary_writer, results=results, step=step, dataset=dataset
)
if is_final_step:
break
del train_dataset
# Can we return a value only on a single rank?
if hvd.rank() == 0:
pbar.close()
logger.info(f"Finished finetuning, job name {run_name}")
return results
def main(args):
#데이터 annotation 파일 경로
annotation_file = args.annotation_file
# 결과 log 및 weight가 저장될 경로
log_dir = os.path.join('logs', '000')
#클래스 파일 경로
classes_path = args.classes_path
class_names = get_classes(classes_path)
num_classes = len(class_names)
# anchors 받아오는 라인
anchors = get_anchors(args.anchors_path)
num_anchors = len(anchors)
# get freeze level according to CLI option
if args.weights_path:
freeze_level = 0
else:
freeze_level = 1
if args.freeze_level is not None:
freeze_level = args.freeze_level
# callbacks for training process
logging = TensorBoard(log_dir=log_dir,
histogram_freq=0,
write_graph=False,
write_grads=False,
write_images=False,
update_freq='batch')
checkpoint = ModelCheckpoint(os.path.join(
log_dir, 'ep{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}.h5'),
monitor='val_loss',
mode='min',
verbose=1,
save_weights_only=False,
save_best_only=True,
period=1)
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
mode='min',
patience=10,
verbose=1,
cooldown=0,
min_lr=1e-10)
early_stopping = EarlyStopping(monitor='val_loss',
min_delta=0,
patience=50,
verbose=1,
mode='min')
terminate_on_nan = TerminateOnNaN()
callbacks = [
logging, checkpoint, reduce_lr, early_stopping, terminate_on_nan
]
# 데이터셋 로딩
dataset = get_dataset(annotation_file)
if args.val_annotation_file:
val_dataset = get_dataset(args.val_annotation_file)
num_train = len(dataset)
num_val = len(val_dataset)
dataset.extend(val_dataset)
else:
val_split = args.val_split
num_val = int(len(dataset) * val_split)
num_train = len(dataset) - num_val
# assign multiscale interval
if args.multiscale:
rescale_interval = args.rescale_interval
else:
rescale_interval = -1 #Doesn't rescale
# model input shape check
input_shape = args.model_image_size
assert (input_shape[0] % 32 == 0 and input_shape[1] % 32
== 0), 'model_image_size should be multiples of 32'
# 모델종류에 따른 data generator 및 모델 생성
if num_anchors == 9:
# YOLOv3 use 9 anchors
get_train_model = get_yolo3_train_model
data_generator = yolo3_data_generator_wrapper
tiny_version = False
elif num_anchors == 6:
# Tiny YOLOv3 use 6 anchors
get_train_model = get_yolo3_train_model
data_generator = yolo3_data_generator_wrapper
tiny_version = True
elif num_anchors == 5:
# YOLOv2 use 5 anchors
get_train_model = get_yolo2_train_model
data_generator = yolo2_data_generator_wrapper
tiny_version = False
else:
raise ValueError('Unsupported anchors number')
# prepare online evaluation callback
if args.eval_online:
eval_callback = EvalCallBack(
args.model_type,
dataset[num_train:],
anchors,
class_names,
args.model_image_size,
args.model_pruning,
log_dir,
eval_epoch_interval=args.eval_epoch_interval,
save_eval_checkpoint=args.save_eval_checkpoint,
elim_grid_sense=args.elim_grid_sense)
callbacks.append(eval_callback)
# prepare train/val data shuffle callback
if args.data_shuffle:
shuffle_callback = DatasetShuffleCallBack(dataset)
callbacks.append(shuffle_callback)
# prepare model pruning config
pruning_end_step = np.ceil(1.0 * num_train / args.batch_size).astype(
np.int32) * args.total_epoch
if args.model_pruning:
pruning_callbacks = [
sparsity.UpdatePruningStep(),
sparsity.PruningSummaries(log_dir=log_dir, profile_batch=0)
]
callbacks = callbacks + pruning_callbacks
# prepare optimizer
optimizer = get_optimizer(args.optimizer,
args.learning_rate,
decay_type=None)
# support multi-gpu training
if args.gpu_num >= 2:
# devices_list=["/gpu:0", "/gpu:1"]
devices_list = ["/gpu:{}".format(n) for n in range(args.gpu_num)]
strategy = tf.distribute.MirroredStrategy(devices=devices_list)
print('Number of devices: {}'.format(strategy.num_replicas_in_sync))
with strategy.scope():
# get multi-gpu train model
model = get_train_model(args.model_type,
anchors,
num_classes,
weights_path=args.weights_path,
freeze_level=freeze_level,
optimizer=optimizer,
label_smoothing=args.label_smoothing,
elim_grid_sense=args.elim_grid_sense,
model_pruning=args.model_pruning,
pruning_end_step=pruning_end_step)
else:
# get normal train model
model = get_train_model(args.model_type,
anchors,
num_classes,
weights_path=args.weights_path,
freeze_level=freeze_level,
optimizer=optimizer,
label_smoothing=args.label_smoothing,
elim_grid_sense=args.elim_grid_sense,
model_pruning=args.model_pruning,
pruning_end_step=pruning_end_step)
model.summary()
# Transfer training some epochs with frozen layers first if needed, to get a stable loss.
initial_epoch = args.init_epoch
epochs = initial_epoch + args.transfer_epoch
print("Transfer training stage")
print(
'Train on {} samples, val on {} samples, with batch size {}, input_shape {}.'
.format(num_train, num_val, args.batch_size, input_shape))
# 성능향상을 위해 초반 일부 epoch은 Transfer Learning 진행 (Initial Epoch ~ Transfer Epoch)
model.fit_generator(
data_generator(dataset[:num_train],
args.batch_size,
input_shape,
anchors,
num_classes,
args.enhance_augment,
rescale_interval,
multi_anchor_assign=args.multi_anchor_assign),
steps_per_epoch=max(1, num_train // args.batch_size),
#validation_data=val_data_generator,
validation_data=data_generator(
dataset[num_train:],
args.batch_size,
input_shape,
anchors,
num_classes,
multi_anchor_assign=args.multi_anchor_assign),
validation_steps=max(1, num_val // args.batch_size),
epochs=epochs,
initial_epoch=initial_epoch,
#verbose=1,
workers=1,
use_multiprocessing=False,
max_queue_size=10,
callbacks=callbacks)
# Wait 2 seconds for next stage
time.sleep(2)
if args.decay_type:
# rebuild optimizer to apply learning rate decay, only after
# unfreeze all layers
callbacks.remove(reduce_lr)
steps_per_epoch = max(1, num_train // args.batch_size)
decay_steps = steps_per_epoch * (args.total_epoch - args.init_epoch -
args.transfer_epoch)
optimizer = get_optimizer(args.optimizer,
args.learning_rate,
decay_type=args.decay_type,
decay_steps=decay_steps)
# Unfreeze the whole network for further tuning
# NOTE: more GPU memory is required after unfreezing the body
print("Unfreeze and continue training, to fine-tune.")
if args.gpu_num >= 2:
with strategy.scope():
for i in range(len(model.layers)):
model.layers[i].trainable = True
model.compile(optimizer=optimizer,
loss={
'yolo_loss': lambda y_true, y_pred: y_pred
}) # recompile to apply the change
else:
for i in range(len(model.layers)):
model.layers[i].trainable = True
model.compile(optimizer=optimizer,
loss={
'yolo_loss': lambda y_true, y_pred: y_pred
}) # recompile to apply the change
print(
'Train on {} samples, val on {} samples, with batch size {}, input_shape {}.'
.format(num_train, num_val, args.batch_size, input_shape))
# Transfer Learning 이후 나머지 Epoch에 대하여 학습 진행 (Transfer Epoch ~ Total Epoch)
# 이 부분이 필요없거나 학습 시간이 너무 오래 걸릴 경우 Total Epoch을 Transfer와 동일하게 두고, 아래 학습을 진행하지 않고 넘어갈 수 있음
# 본인 컴퓨터 사양에 맞춰서 진행
model.fit_generator(
data_generator(dataset[:num_train],
args.batch_size,
input_shape,
anchors,
num_classes,
args.enhance_augment,
rescale_interval,
multi_anchor_assign=args.multi_anchor_assign),
steps_per_epoch=max(1, num_train // args.batch_size),
#validation_data=val_data_generator,
validation_data=data_generator(
dataset[num_train:],
args.batch_size,
input_shape,
anchors,
num_classes,
multi_anchor_assign=args.multi_anchor_assign),
validation_steps=max(1, num_val // args.batch_size),
epochs=args.total_epoch,
initial_epoch=epochs,
#verbose=1,
workers=1,
use_multiprocessing=False,
max_queue_size=10,
callbacks=callbacks)
# Finally store model
if args.model_pruning:
model = sparsity.strip_pruning(model)
model.save(os.path.join(log_dir, 'trained_final.h5'))
def main(args):
annotation_file = args.annotation_file
classes_path = args.classes_path
class_names = get_classes(classes_path)
num_classes = len(class_names)
anchors = get_anchors(args.anchors_path)
num_anchors = len(anchors)
log_dir_path = args.log_directory
try:
log_dir = os.path.join('logs', log_dir_path)
except TypeError:
date_now = datetime.now()
log_dir_folder_name = f'{date_now.strftime("%Y_%m_%d_%H%M%S")}_{args.model_type}_TransferEp_{args.transfer_epoch}_TotalEP_{args.total_epoch}'
log_dir = os.path.realpath(os.path.join(
'logs',
log_dir_folder_name
))
# get freeze level according to CLI option
if args.weights_path:
freeze_level = 0
else:
freeze_level = 1
if args.freeze_level is not None:
freeze_level = args.freeze_level
# How many percentage of layers to unfreeze in fine tuning
unfreeze_level = args.unfreeze_level
# callbacks for training process
logging = TensorBoard(log_dir=log_dir, histogram_freq=0, write_graph=False, write_grads=False, write_images=False,
update_freq='batch')
checkpoint = ModelCheckpoint(
filepath=log_dir + os.sep + 'ep{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}.h5',
monitor='val_loss',
mode='min',
verbose=1,
save_weights_only=False,
save_best_only=True,
period=1
)
reduce_lr = ReduceLROnPlateau(
monitor='val_loss',
factor=0.5, mode='min',
patience=10,
verbose=1,
cooldown=0,
min_lr=1e-10
)
early_stopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=50, verbose=1, mode='min')
terminate_on_nan = TerminateOnNaN()
callbacks = [logging, checkpoint, reduce_lr, early_stopping, terminate_on_nan]
# get train&val dataset
dataset = get_dataset(annotation_file)
if args.val_annotation_file:
val_dataset = get_dataset(args.val_annotation_file)
num_train = len(dataset)
num_val = len(val_dataset)
dataset.extend(val_dataset)
else:
val_split = args.val_split
num_val = int(len(dataset) * val_split)
num_train = len(dataset) - num_val
# assign multiscale interval
if args.multiscale:
rescale_interval = args.rescale_interval
else:
rescale_interval = -1 # Doesn't rescale
# model input shape check
input_shape = args.model_image_size
assert (input_shape[0] % 32 == 0 and input_shape[1] % 32 == 0), 'model_image_size should be multiples of 32'
# get different model type & train&val data generator
if num_anchors == 9:
# YOLOv3 use 9 anchors
get_train_model = get_yolo3_train_model
data_generator = yolo3_data_generator_wrapper
# tf.keras.Sequence style data generator
# train_data_generator = Yolo3DataGenerator(dataset[:num_train], args.batch_size, input_shape, anchors, num_classes, args.enhance_augment, rescale_interval, args.multi_anchor_assign)
# val_data_generator = Yolo3DataGenerator(dataset[num_train:], args.batch_size, input_shape, anchors, num_classes, multi_anchor_assign=args.multi_anchor_assign)
tiny_version = False
elif num_anchors == 6:
# Tiny YOLOv3 use 6 anchors
get_train_model = get_yolo3_train_model
data_generator = yolo3_data_generator_wrapper
# tf.keras.Sequence style data generator
# train_data_generator = Yolo3DataGenerator(dataset[:num_train], args.batch_size, input_shape, anchors, num_classes, args.enhance_augment, rescale_interval, args.multi_anchor_assign)
# val_data_generator = Yolo3DataGenerator(dataset[num_train:], args.batch_size, input_shape, anchors, num_classes, multi_anchor_assign=args.multi_anchor_assign)
tiny_version = True
elif num_anchors == 5:
# YOLOv2 use 5 anchors
get_train_model = get_yolo2_train_model
data_generator = yolo2_data_generator_wrapper
# tf.keras.Sequence style data generator
# train_data_generator = Yolo2DataGenerator(dataset[:num_train], args.batch_size, input_shape, anchors, num_classes, args.enhance_augment, rescale_interval)
# val_data_generator = Yolo2DataGenerator(dataset[num_train:], args.batch_size, input_shape, anchors, num_classes)
tiny_version = False
else:
raise ValueError('Unsupported anchors number')
# prepare online evaluation callback
if args.eval_online:
eval_callback = EvalCallBack(
model_type=args.model_type,
annotation_lines=dataset[num_train:],
anchors=anchors,
class_names=class_names,
model_image_size=args.model_image_size,
model_pruning=args.model_pruning,
log_dir=log_dir,
eval_epoch_interval=args.eval_epoch_interval,
save_eval_checkpoint=args.save_eval_checkpoint,
elim_grid_sense=args.elim_grid_sense
)
callbacks.append(eval_callback)
# prepare train/val data shuffle callback
if args.data_shuffle:
shuffle_callback = DatasetShuffleCallBack(dataset)
callbacks.append(shuffle_callback)
# prepare model pruning config
pruning_end_step = np.ceil(1.0 * num_train / args.batch_size).astype(np.int32) * args.total_epoch
if args.model_pruning:
pruning_callbacks = [sparsity.UpdatePruningStep(), sparsity.PruningSummaries(log_dir=log_dir, profile_batch=0)]
callbacks = callbacks + pruning_callbacks
# prepare optimizer
optimizer = get_optimizer(args.optimizer, args.learning_rate, decay_type=None)
# support multi-gpu training
if args.gpu_num >= 2:
# devices_list=["/gpu:0", "/gpu:1"]
devices_list = ["/gpu:{}".format(n) for n in range(args.gpu_num)]
strategy = tf.distribute.MirroredStrategy(devices=devices_list)
print('Number of devices: {}'.format(strategy.num_replicas_in_sync))
with strategy.scope():
# get multi-gpu train model
model = get_train_model(
model_type=args.model_type,
anchors=anchors,
num_classes=num_classes,
weights_path=args.weights_path,
freeze_level=freeze_level,
optimizer=optimizer,
label_smoothing=args.label_smoothing,
elim_grid_sense=args.elim_grid_sense,
model_pruning=args.model_pruning,
pruning_end_step=pruning_end_step
)
else:
# get normal train model
model = get_train_model(
model_type=args.model_type,
anchors=anchors,
num_classes=num_classes,
weights_path=args.weights_path,
freeze_level=freeze_level,
optimizer=optimizer,
label_smoothing=args.label_smoothing,
elim_grid_sense=args.elim_grid_sense,
model_pruning=args.model_pruning,
pruning_end_step=pruning_end_step
)
if args.show_history:
model.summary()
layers_count = len(model.layers)
print(f'Total layers: {layers_count}')
# Transfer training some epochs with frozen layers first if needed, to get a stable loss.
initial_epoch = args.init_epoch
epochs = initial_epoch + args.transfer_epoch
print("Transfer training stage")
print('Train on {} samples, val on {} samples, with batch size {}, input_shape {}.'.format(num_train, num_val,
args.batch_size,
input_shape))
# model.fit_generator(train_data_generator,
"""
Transfer training steps, train with freeze layers
"""
model.fit(
data_generator(
annotation_lines=dataset[:num_train],
batch_size=args.batch_size,
input_shape=input_shape,
anchors=anchors,
num_classes=num_classes,
enhance_augment=args.enhance_augment,
rescale_interval=rescale_interval,
multi_anchor_assign=args.multi_anchor_assign
),
steps_per_epoch=max(1, num_train // args.batch_size),
# validation_data=val_data_generator,
validation_data=data_generator(
annotation_lines=dataset[num_train:],
batch_size=args.batch_size,
input_shape=input_shape,
anchors=anchors,
num_classes=num_classes,
multi_anchor_assign=args.multi_anchor_assign
),
validation_steps=max(1, num_val // args.batch_size),
epochs=epochs,
initial_epoch=initial_epoch,
# verbose=1,
workers=1,
use_multiprocessing=False,
max_queue_size=10,
callbacks=callbacks
)
# Wait 2 seconds for next stage
time.sleep(2)
if args.decay_type:
# rebuild optimizer to apply learning rate decay, only after
# unfreeze all layers
callbacks.remove(reduce_lr)
steps_per_epoch = max(1, num_train // args.batch_size)
decay_steps = steps_per_epoch * (args.total_epoch - args.init_epoch - args.transfer_epoch)
optimizer = get_optimizer(args.optimizer, args.learning_rate, decay_type=args.decay_type,
decay_steps=decay_steps)
# Unfreeze the whole network for further tuning
# NOTE: more GPU memory is required after unfreezing the body
fine_tune_layers = int(layers_count * unfreeze_level)
print(f"Unfreeze {unfreeze_level * 100}% of layers and continue training, to fine-tune.")
print(f"Unfroze {fine_tune_layers} layers of {layers_count}")
if args.gpu_num >= 2:
with strategy.scope():
for i in range(layers_count - fine_tune_layers, layers_count):
model.layers[i].trainable = True
model.compile(optimizer=optimizer,
loss={'yolo_loss': lambda y_true, y_pred: y_pred}) # recompile to apply the change
else:
for i in range(layers_count - fine_tune_layers, layers_count):
model.layers[i].trainable = True
model.compile(optimizer=optimizer,
loss={'yolo_loss': lambda y_true, y_pred: y_pred}) # recompile to apply the change
print('Train on {} samples, val on {} samples, with batch size {}, input_shape {}.'.format(num_train, num_val,
args.batch_size,
input_shape))
"""
Fine-tuning steps, more memory will be used. LR (Learning Rate) will be decayed
"""
# model.fit_generator(train_data_generator,
model.fit(
# The YOLO data augmentation generator tool
data_generator(
annotation_lines=dataset[:num_train],
batch_size=args.batch_size,
input_shape=input_shape,
anchors=anchors,
num_classes=num_classes,
enhance_augment=args.enhance_augment,
rescale_interval=rescale_interval,
multi_anchor_assign=args.multi_anchor_assign
),
steps_per_epoch=max(1, num_train // args.batch_size),
# validation_data=val_data_generator,
# Validation generator
validation_data=data_generator(
annotation_lines=dataset[num_train:],
batch_size=args.batch_size,
input_shape=input_shape,
anchors=anchors,
num_classes=num_classes,
multi_anchor_assign=args.multi_anchor_assign
),
validation_steps=max(1, num_val // args.batch_size),
epochs=args.total_epoch,
initial_epoch=epochs,
# verbose=1,
workers=1,
use_multiprocessing=False,
max_queue_size=10,
callbacks=callbacks
)
# Finally store model
if args.model_pruning:
model = sparsity.strip_pruning(model)
model.save(os.path.join(log_dir, 'trained_final.h5'))
def main(args):
annotation_file = args.annotation_file
log_dir = os.path.join('logs', '000')
classes_path = args.classes_path
class_names = get_classes(classes_path)
num_classes = len(class_names)
anchors = get_anchors(args.anchors_path)
num_anchors = len(anchors)
# get freeze level according to CLI option
if args.weights_path:
freeze_level = 0
else:
freeze_level = 1
if args.freeze_level is not None:
freeze_level = args.freeze_level
# callbacks for training process
logging = TensorBoard(log_dir=log_dir,
histogram_freq=0,
write_graph=False,
write_grads=False,
write_images=False,
update_freq='batch')
checkpoint = ModelCheckpoint(os.path.join(
log_dir, 'ep{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}.h5'),
monitor='val_loss',
mode='min',
verbose=1,
save_weights_only=False,
save_best_only=True,
period=1)
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
mode='min',
patience=10,
verbose=1,
cooldown=0,
min_lr=1e-10)
early_stopping = EarlyStopping(monitor='val_loss',
min_delta=0,
patience=50,
verbose=1,
mode='min')
terminate_on_nan = TerminateOnNaN()
callbacks = [
logging, checkpoint, reduce_lr, early_stopping, terminate_on_nan
]
# get train&val dataset
dataset = get_dataset(annotation_file)
if args.val_annotation_file:
val_dataset = get_dataset(args.val_annotation_file)
num_train = len(dataset)
num_val = len(val_dataset)
dataset.extend(val_dataset)
else:
val_split = args.val_split
num_val = int(len(dataset) * val_split)
num_train = len(dataset) - num_val
# assign multiscale interval
if args.multiscale:
rescale_interval = args.rescale_interval
else:
rescale_interval = -1 #Doesn't rescale
# model input shape check
input_shape = args.model_image_size
assert (input_shape[0] % 32 == 0 and input_shape[1] % 32
== 0), 'model_image_size should be multiples of 32'
# get different model type & train&val data generator
if args.model_type.startswith(
'scaled_yolo4_') or args.model_type.startswith('yolo5_'):
# Scaled-YOLOv4 & YOLOv5 entrance, use yolo5 submodule but now still yolo3 data generator
# TODO: create new yolo5 data generator to apply YOLOv5 anchor assignment
get_train_model = get_yolo5_train_model
data_generator = yolo5_data_generator_wrapper
# tf.keras.Sequence style data generator
#train_data_generator = Yolo5DataGenerator(dataset[:num_train], args.batch_size, input_shape, anchors, num_classes, args.enhance_augment, rescale_interval, args.multi_anchor_assign)
#val_data_generator = Yolo5DataGenerator(dataset[num_train:], args.batch_size, input_shape, anchors, num_classes, multi_anchor_assign=args.multi_anchor_assign)
tiny_version = False
elif args.model_type.startswith('yolo3_') or args.model_type.startswith(
'yolo4_'):
#if num_anchors == 9:
# YOLOv3 & v4 entrance, use 9 anchors
get_train_model = get_yolo3_train_model
data_generator = yolo3_data_generator_wrapper
# tf.keras.Sequence style data generator
#train_data_generator = Yolo3DataGenerator(dataset[:num_train], args.batch_size, input_shape, anchors, num_classes, args.enhance_augment, rescale_interval, args.multi_anchor_assign)
#val_data_generator = Yolo3DataGenerator(dataset[num_train:], args.batch_size, input_shape, anchors, num_classes, multi_anchor_assign=args.multi_anchor_assign)
tiny_version = False
elif args.model_type.startswith(
'tiny_yolo3_') or args.model_type.startswith('tiny_yolo4_'):
#elif num_anchors == 6:
# Tiny YOLOv3 & v4 entrance, use 6 anchors
get_train_model = get_yolo3_train_model
data_generator = yolo3_data_generator_wrapper
# tf.keras.Sequence style data generator
#train_data_generator = Yolo3DataGenerator(dataset[:num_train], args.batch_size, input_shape, anchors, num_classes, args.enhance_augment, rescale_interval, args.multi_anchor_assign)
#val_data_generator = Yolo3DataGenerator(dataset[num_train:], args.batch_size, input_shape, anchors, num_classes, multi_anchor_assign=args.multi_anchor_assign)
tiny_version = True
elif args.model_type.startswith('yolo2_') or args.model_type.startswith(
'tiny_yolo2_'):
#elif num_anchors == 5:
# YOLOv2 & Tiny YOLOv2 use 5 anchors
get_train_model = get_yolo2_train_model
data_generator = yolo2_data_generator_wrapper
# tf.keras.Sequence style data generator
#train_data_generator = Yolo2DataGenerator(dataset[:num_train], args.batch_size, input_shape, anchors, num_classes, args.enhance_augment, rescale_interval)
#val_data_generator = Yolo2DataGenerator(dataset[num_train:], args.batch_size, input_shape, anchors, num_classes)
tiny_version = False
else:
raise ValueError('Unsupported model type')
# prepare online evaluation callback
if args.eval_online:
eval_callback = EvalCallBack(
args.model_type,
dataset[num_train:],
anchors,
class_names,
args.model_image_size,
args.model_pruning,
log_dir,
eval_epoch_interval=args.eval_epoch_interval,
save_eval_checkpoint=args.save_eval_checkpoint,
elim_grid_sense=args.elim_grid_sense)
callbacks.append(eval_callback)
# prepare train/val data shuffle callback
if args.data_shuffle:
shuffle_callback = DatasetShuffleCallBack(dataset)
callbacks.append(shuffle_callback)
# prepare model pruning config
pruning_end_step = np.ceil(1.0 * num_train / args.batch_size).astype(
np.int32) * args.total_epoch
if args.model_pruning:
pruning_callbacks = [
sparsity.UpdatePruningStep(),
sparsity.PruningSummaries(log_dir=log_dir, profile_batch=0)
]
callbacks = callbacks + pruning_callbacks
# prepare optimizer
optimizer = get_optimizer(args.optimizer,
args.learning_rate,
average_type=None,
decay_type=None)
# support multi-gpu training
if args.gpu_num >= 2:
# devices_list=["/gpu:0", "/gpu:1"]
devices_list = ["/gpu:{}".format(n) for n in range(args.gpu_num)]
strategy = tf.distribute.MirroredStrategy(devices=devices_list)
print('Number of devices: {}'.format(strategy.num_replicas_in_sync))
with strategy.scope():
# get multi-gpu train model
model = get_train_model(args.model_type,
anchors,
num_classes,
weights_path=args.weights_path,
freeze_level=freeze_level,
optimizer=optimizer,
label_smoothing=args.label_smoothing,
elim_grid_sense=args.elim_grid_sense,
model_pruning=args.model_pruning,
pruning_end_step=pruning_end_step)
else:
# get normal train model
model = get_train_model(args.model_type,
anchors,
num_classes,
weights_path=args.weights_path,
freeze_level=freeze_level,
optimizer=optimizer,
label_smoothing=args.label_smoothing,
elim_grid_sense=args.elim_grid_sense,
model_pruning=args.model_pruning,
pruning_end_step=pruning_end_step)
model.summary()
# Transfer training some epochs with frozen layers first if needed, to get a stable loss.
initial_epoch = args.init_epoch
epochs = initial_epoch + args.transfer_epoch
print("Transfer training stage")
print(
'Train on {} samples, val on {} samples, with batch size {}, input_shape {}.'
.format(num_train, num_val, args.batch_size, input_shape))
#model.fit_generator(train_data_generator,
model.fit_generator(
data_generator(dataset[:num_train],
args.batch_size,
input_shape,
anchors,
num_classes,
args.enhance_augment,
rescale_interval,
multi_anchor_assign=args.multi_anchor_assign),
steps_per_epoch=max(1, num_train // args.batch_size),
#validation_data=val_data_generator,
validation_data=data_generator(
dataset[num_train:],
args.batch_size,
input_shape,
anchors,
num_classes,
multi_anchor_assign=args.multi_anchor_assign),
validation_steps=max(1, num_val // args.batch_size),
epochs=epochs,
initial_epoch=initial_epoch,
#verbose=1,
workers=1,
use_multiprocessing=False,
max_queue_size=10,
callbacks=callbacks)
# Wait 2 seconds for next stage
time.sleep(2)
if args.decay_type or args.average_type:
# rebuild optimizer to apply learning rate decay or weights averager,
# only after unfreeze all layers
if args.decay_type:
callbacks.remove(reduce_lr)
if args.average_type == 'ema' or args.average_type == 'swa':
# weights averager need tensorflow-addons,
# which request TF 2.x and have version compatibility
import tensorflow_addons as tfa
callbacks.remove(checkpoint)
avg_checkpoint = tfa.callbacks.AverageModelCheckpoint(
filepath=os.path.join(
log_dir,
'ep{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}.h5'),
update_weights=True,
monitor='val_loss',
mode='min',
verbose=1,
save_weights_only=False,
save_best_only=True,
period=1)
callbacks.append(avg_checkpoint)
steps_per_epoch = max(1, num_train // args.batch_size)
decay_steps = steps_per_epoch * (args.total_epoch - args.init_epoch -
args.transfer_epoch)
optimizer = get_optimizer(args.optimizer,
args.learning_rate,
average_type=args.average_type,
decay_type=args.decay_type,
decay_steps=decay_steps)
# Unfreeze the whole network for further tuning
# NOTE: more GPU memory is required after unfreezing the body
print("Unfreeze and continue training, to fine-tune.")
if args.gpu_num >= 2:
with strategy.scope():
for i in range(len(model.layers)):
model.layers[i].trainable = True
model.compile(optimizer=optimizer,
loss={
'yolo_loss': lambda y_true, y_pred: y_pred
}) # recompile to apply the change
else:
for i in range(len(model.layers)):
model.layers[i].trainable = True
model.compile(optimizer=optimizer,
loss={
'yolo_loss': lambda y_true, y_pred: y_pred
}) # recompile to apply the change
print(
'Train on {} samples, val on {} samples, with batch size {}, input_shape {}.'
.format(num_train, num_val, args.batch_size, input_shape))
#model.fit_generator(train_data_generator,
model.fit_generator(
data_generator(dataset[:num_train],
args.batch_size,
input_shape,
anchors,
num_classes,
args.enhance_augment,
rescale_interval,
multi_anchor_assign=args.multi_anchor_assign),
steps_per_epoch=max(1, num_train // args.batch_size),
#validation_data=val_data_generator,
validation_data=data_generator(
dataset[num_train:],
args.batch_size,
input_shape,
anchors,
num_classes,
multi_anchor_assign=args.multi_anchor_assign),
validation_steps=max(1, num_val // args.batch_size),
epochs=args.total_epoch,
initial_epoch=epochs,
#verbose=1,
workers=1,
use_multiprocessing=False,
max_queue_size=10,
callbacks=callbacks)
# Finally store model
if args.model_pruning:
model = sparsity.strip_pruning(model)
model.save(os.path.join(log_dir, 'trained_final.h5'))
def main(args):
model_type = "yolo3_darknet_spp" # yolo3_darknet_spp, yolo3_darknet
current_dir = os.path.dirname(__file__) + "/"
print("current_dir == ", current_dir)
annotation_file = current_dir + "sample/trainval/train.txt"
val_annotation_file = current_dir + "sample/trainval/val.txt"
classes_path = current_dir + "sample/trainval/train_classes.txt"
anchors_path = current_dir + "sample/trainval/yolo_anchors.txt"
weights_path = current_dir + "weights/yolov3-spp.h5"
load_weights_path = None # None or "{weights path}"
is_one_stage_train = True
learning_rate_1 = 1e-4
learning_rate_2 = 1e-5
epoch_1 = args.max_epochs_1
epoch_2 = args.max_epochs_2
batch_size_1 = args.batch_size_1
batch_size_2 = args.batch_size_2
freeze_level = 2
model_image_size = (416, 416)
val_split = 0.1
label_smoothing = 0
enhance_augment = None # enhance data augmentation type (None/mosaic)
rescale_interval = 0 # Number of iteration(batches) interval to rescale input size, default=10
log_dir = os.path.join('logs', '20200602')
class_names = get_classes(classes_path)
num_classes = len(class_names)
anchors = get_anchors(anchors_path)
logging = TensorBoard(log_dir=log_dir, update_freq='batch')
checkpoint = ModelCheckpoint(os.path.join(
log_dir, 'ep{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}.h5'),
monitor='val_loss',
verbose=1,
save_weights_only=True,
save_best_only=True,
period=1)
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=10,
verbose=1,
cooldown=0,
min_lr=1e-10)
early_stopping = EarlyStopping(monitor='val_loss',
min_delta=0,
patience=50,
verbose=1)
# terminate_on_nan = TerminateOnNaN()
callbacks = [
logging, checkpoint, reduce_lr, early_stopping,
ModertFileToObs(log_dir, args)
]
# callbacks = [logging, checkpoint, reduce_lr]
# get train&val dataset
dataset = get_dataset(annotation_file)
dataset = [current_dir + d for d in dataset]
if val_annotation_file != "":
val_dataset = get_dataset(val_annotation_file)
num_train = len(dataset)
num_val = len(val_dataset)
dataset.extend(val_dataset)
else:
val_split = val_split
num_val = int(len(dataset) * val_split)
num_train = len(dataset) - num_val
# num_val = 100
# num_train = 200
# model input shape check
input_shape = model_image_size
assert (input_shape[0] % 32 == 0
and input_shape[1] % 32 == 0), 'Multiples of 32 required'
get_train_model = get_yolo3_train_model
data_generator = yolo3_data_generator_wrapper
# get train model
model = get_train_model(model_type,
anchors,
num_classes,
input_shape,
weights_path=weights_path,
freeze_level=freeze_level,
label_smoothing=label_smoothing)
if load_weights_path:
model.load_weights(load_weights_path)
print("reload weights: {}".format(load_weights_path))
if is_one_stage_train:
model.compile(optimizer=get_optimizer(learning_rate_1),
loss={
'yolo_loss': lambda y_true, y_pred: y_pred
})
print(
'One stage Train on {} samples, val on {} samples, with batch size {}, '
'input_shape {}.'.format(num_train, num_val, batch_size_1,
input_shape))
model.fit_generator(
data_generator(dataset[:num_train], batch_size_1, input_shape,
anchors, num_classes, enhance_augment),
steps_per_epoch=max(1, num_train // batch_size_1),
validation_data=data_generator(dataset[:num_val], batch_size_1,
input_shape, anchors, num_classes),
validation_steps=max(1, num_val // batch_size_1),
epochs=epoch_1,
initial_epoch=0,
workers=1,
use_multiprocessing=False,
max_queue_size=10,
callbacks=callbacks)
model.save_weights(os.path.join(log_dir, 'trained_weights_stage_1.h5'))
if True:
print("Unfreeze and continue training, to fine-tune.")
for i in range(len(model.layers)):
model.layers[i].trainable = True
model.compile(optimizer=get_optimizer(learning_rate_2),
loss={
'yolo_loss': lambda y_true, y_pred: y_pred
})
print(
'Two stage Train on {} samples, val on {} samples, with batch size {}, input_shape {}.'
.format(num_train, num_val, batch_size_2, input_shape))
model.fit_generator(data_generator(dataset[:num_train], batch_size_2,
input_shape, anchors, num_classes,
enhance_augment, rescale_interval),
steps_per_epoch=max(1, num_train // batch_size_2),
validation_data=data_generator(
dataset[:num_val], batch_size_2, input_shape,
anchors, num_classes),
validation_steps=max(1, num_val // batch_size_2),
epochs=epoch_2,
initial_epoch=epoch_1,
workers=1,
use_multiprocessing=False,
max_queue_size=10,
callbacks=callbacks)
model.save_weights(os.path.join(log_dir, 'trained_weights_final.h5'))
gen_model_dir(log_dir, args, classes_path, anchors_path)
