print(count)
pos = batch_filenames[0].rfind('/')
image_name = batch_filenames[0][pos + 1:len(batch_filenames[0])]
time_start = time.time()
y_pred = model.predict(batch_images)
time_end = time.time()
print('totally cost:', time_end - time_start)
normalize_coords = True
# y_pred_decoded = decode_detections_fast(y_pred,
# confidence_thresh=cfgs.FILTERED_SCORES,
# iou_threshold=cfgs.NMS_IOU_THRESHOLD,
# top_k=cfgs.MAXIMUM_DETECTIONS)
y_pred_decoded_inv = apply_inverse_transforms(y_pred,
batch_inverse_transforms)
scores = np.ones(shape=[
len(batch_original_labels[0]),
], dtype=np.float32) * cfgs.ONLY_DRAW_BOXES
gt_img = draw_box_in_img.draw_boxes_with_label_and_scores(
batch_images[0], batch_original_labels[0][:, -4:],
batch_original_labels[0][:, 0], scores)
gt_img = cv2.resize(gt_img, dsize=(800, 600))
cv2.namedWindow("Image")
cv2.imshow('Image', gt_img)
# cv2.waitKey()
result_img = draw_box_in_img.draw_boxes_with_label_and_scores(
batch_images[0], y_pred_decoded_inv[0][:, -4:],
y_pred_decoded_inv[0][:, 0], y_pred_decoded_inv[0][:, 1])
result_img = cv2.resize(result_img, dsize=(800, 600))
def main():
create_new_model = True if args.model_name == 'default' else False
if create_new_model:
K.clear_session() # Clear previous models from memory.
model = ssd_512(image_size=(Config.img_height, Config.img_width,
Config.img_channels),
n_classes=Config.n_classes,
mode='training',
l2_regularization=Config.l2_regularization,
scales=Config.scales,
aspect_ratios_per_layer=Config.aspect_ratios,
two_boxes_for_ar1=Config.two_boxes_for_ar1,
steps=Config.steps,
offsets=Config.offsets,
clip_boxes=Config.clip_boxes,
variances=Config.variances,
normalize_coords=Config.normalize_coords,
subtract_mean=Config.mean_color,
swap_channels=Config.swap_channels)
adam = Adam(lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
decay=0.0)
ssd_loss = SSDLoss(neg_pos_ratio=3, alpha=1.0)
model.compile(optimizer=adam, loss=ssd_loss.compute_loss)
else:
model_path = "weights/" + args.model_name + ".h5"
# We need to create an SSDLoss object in order to pass that to the model loader.
ssd_loss = SSDLoss(neg_pos_ratio=3, alpha=1.0)
K.clear_session() # Clear previous models from memory.
model = load_model(model_path,
custom_objects={
'AnchorBoxes': AnchorBoxes,
'L2Normalization': L2Normalization,
'compute_loss': ssd_loss.compute_loss
})
# Load the data
train_dataset = DataGenerator(load_images_into_memory=True,
hdf5_dataset_path=os.getcwd() + "/data/" +
args.dataset + '/polyp_train.h5')
val_dataset = DataGenerator(load_images_into_memory=True,
hdf5_dataset_path=os.getcwd() + "/data/" +
args.dataset + '/polyp_val.h5')
train_dataset_size = train_dataset.get_dataset_size()
val_dataset_size = val_dataset.get_dataset_size()
print("Number of images in the training dataset:\t{:>6}".format(
train_dataset_size))
print("Number of images in the validation dataset:\t{:>6}".format(
val_dataset_size))
batch_size = args.batch_size
# For the training generator:
ssd_data_augmentation = SSDDataAugmentation(img_height=Config.img_height,
img_width=Config.img_width,
background=Config.mean_color)
# For the validation generator:
convert_to_3_channels = ConvertTo3Channels()
resize = Resize(height=Config.img_height, width=Config.img_width)
# 5: Instantiate an encoder that can encode ground truth labels into the format needed by the SSD loss function.
# The encoder constructor needs the spatial dimensions of the model's predictor layers to create the anchor boxes.
predictor_sizes = [
model.get_layer('conv4_3_norm_mbox_conf').output_shape[1:3],
model.get_layer('fc7_mbox_conf').output_shape[1:3],
model.get_layer('conv6_2_mbox_conf').output_shape[1:3],
model.get_layer('conv7_2_mbox_conf').output_shape[1:3],
model.get_layer('conv8_2_mbox_conf').output_shape[1:3],
model.get_layer('conv9_2_mbox_conf').output_shape[1:3],
model.get_layer('conv10_2_mbox_conf').output_shape[1:3]
]
ssd_input_encoder = SSDInputEncoder(
img_height=Config.img_height,
img_width=Config.img_width,
n_classes=Config.n_classes,
predictor_sizes=predictor_sizes,
scales=Config.scales,
aspect_ratios_per_layer=Config.aspect_ratios,
two_boxes_for_ar1=Config.two_boxes_for_ar1,
steps=Config.steps,
offsets=Config.offsets,
clip_boxes=Config.clip_boxes,
variances=Config.variances,
matching_type='multi',
pos_iou_threshold=0.5,
neg_iou_limit=0.5,
normalize_coords=Config.normalize_coords)
# 6: Create the generator handles that will be passed to Keras' `fit_generator()` function.
train_generator = train_dataset.generate(
batch_size=batch_size,
shuffle=True,
transformations=[ssd_data_augmentation],
label_encoder=ssd_input_encoder,
returns={'processed_images', 'encoded_labels'},
keep_images_without_gt=False)
val_generator = val_dataset.generate(
batch_size=batch_size,
shuffle=False,
transformations=[convert_to_3_channels, resize],
label_encoder=ssd_input_encoder,
returns={'processed_images', 'encoded_labels'},
keep_images_without_gt=False)
model_checkpoint = ModelCheckpoint(
filepath=os.getcwd() +
'/weights/ssd512_epoch-{epoch:02d}_loss-{loss:.4f}_val_loss-{val_loss:.4f}.h5',
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='auto',
period=30)
csv_logger = CSVLogger(filename='ssd512_training_log.csv',
separator=',',
append=True)
learning_rate_scheduler = LearningRateScheduler(schedule=lr_schedule)
terminate_on_nan = TerminateOnNaN()
tf_log = keras.callbacks.TensorBoard(log_dir=TF_LOG_PATH + args.tf_logs,
histogram_freq=0,
batch_size=batch_size,
write_graph=True,
write_grads=False,
write_images=False)
callbacks = [
model_checkpoint, csv_logger, learning_rate_scheduler,
terminate_on_nan, tf_log
]
# If you're resuming a previous training, set `initial_epoch` and `final_epoch` accordingly.
initial_epoch = 0
final_epoch = args.final_epoch
steps_per_epoch = 500
# Train/Fit the model
if args.predict_mode == 'train':
history = model.fit_generator(generator=train_generator,
steps_per_epoch=steps_per_epoch,
epochs=final_epoch,
callbacks=callbacks,
validation_data=val_generator,
validation_steps=ceil(val_dataset_size /
batch_size),
initial_epoch=initial_epoch)
# Prediction Output
predict_generator = val_dataset.generate(
batch_size=1,
shuffle=True,
transformations=[convert_to_3_channels, resize],
label_encoder=None,
returns={
'processed_images', 'filenames', 'inverse_transform',
'original_images', 'original_labels'
},
keep_images_without_gt=False)
i = 0
for val in range(val_dataset_size):
batch_images, batch_filenames, batch_inverse_transforms, batch_original_images, batch_original_labels = next(
predict_generator)
y_pred = model.predict(batch_images)
y_pred_decoded = decode_detections(
y_pred,
confidence_thresh=0.5,
iou_threshold=0.4,
top_k=200,
normalize_coords=Config.normalize_coords,
img_height=Config.img_height,
img_width=Config.img_width)
# 5: Convert the predictions for the original image.
y_pred_decoded_inv = apply_inverse_transforms(
y_pred_decoded, batch_inverse_transforms)
np.set_printoptions(precision=2, suppress=True, linewidth=90)
print("Predicted boxes:\n")
print(' class conf xmin ymin xmax ymax')
print(y_pred_decoded_inv[i])
plt.figure(figsize=(20, 12))
plt.imshow(batch_images[i])
current_axis = plt.gca()
colors = plt.cm.hsv(
np.linspace(0, 1, Config.n_classes +
1)).tolist() # Set the colors for the bounding boxes
classes = [
'background', 'polyps'
] # Just so we can print class names onto the image instead of IDs
for box in batch_original_labels[i]:
xmin = box[1]
ymin = box[2]
xmax = box[3]
ymax = box[4]
label = '{}'.format(classes[int(box[0])])
current_axis.add_patch(
plt.Rectangle((xmin, ymin),
xmax - xmin,
ymax - ymin,
color='green',
fill=False,
linewidth=2))
current_axis.text(xmin,
ymin,
label,
size='x-large',
color='white',
bbox={
'facecolor': 'green',
'alpha': 1.0
})
for box in y_pred_decoded_inv[i]:
xmin = box[2]
ymin = box[3]
xmax = box[4]
ymax = box[5]
color = colors[int(box[0])]
label = '{}: {:.2f}'.format(classes[int(box[0])], box[1])
current_axis.add_patch(
plt.Rectangle((xmin, ymin),
xmax - xmin,
ymax - ymin,
color=color,
fill=False,
linewidth=2))
current_axis.text(xmin,
ymin,
label,
size='x-large',
color='white',
bbox={
'facecolor': color,
'alpha': 1.0
})
image = plt.gcf()
plt.draw()
image.savefig(os.getcwd() + "/val_ssd512val_" + str(val) + ".png",
dpi=100)
def test_config(config):
'''
Test the given configuration ; the configuration should already have been
used for training purposes, or this will return an error (see ssd_train.py)
Arguments:
config : the configuration of the model to use ; should already be
loaded.
'''
local_dir = config.ROOT_FOLDER
data_dir = config.DATA_DIR
img_shape = config.IMG_SHAPE
img_height = img_shape[0] # Height of the model input images
img_width = img_shape[1] # Width of the model input images
img_channels = img_shape[
2] # Number of color channels of the model input images
n_classes = 20 # Number of positive classes, e.g. 20 for Pascal VOC, 80 for MS COCO
normalize_coords = True
K.clear_session() # Clear previous models from memory.
print("[INFO] loading model...")
model_path = os.path.join(local_dir, 'models', config.MODEL_NAME)
# We need to create an SSDLoss object in order to pass that to the model loader.
ssd_loss = SSDLoss(neg_pos_ratio=3, n_neg_min=0, alpha=1.0)
model = load_model(model_path,
custom_objects={
'AnchorBoxes': AnchorBoxes,
'L2Normalization': L2Normalization,
'DecodeDetections': DecodeDetections,
'compute_loss': ssd_loss.compute_loss
})
classes = config.CLASSES
dataset = DataGenerator(load_images_into_memory=False,
hdf5_dataset_path=None)
dataset_images_dir = os.path.join(data_dir, 'Images')
dataset_annotations_dir = os.path.join(data_dir, 'Annotations/')
dataset_test_image_set_filename = os.path.join(data_dir,
'ImageSets\\test.txt')
dataset.parse_xml(images_dirs=[dataset_images_dir],
image_set_filenames=[dataset_test_image_set_filename],
annotations_dirs=[dataset_annotations_dir],
classes=classes,
include_classes='all',
exclude_truncated=False,
exclude_difficult=True,
ret=False)
dataset.create_hdf5_dataset(file_path=config.MODEL_NAME,
resize=False,
variable_image_size=True,
verbose=True)
convert_to_3_channels = ConvertTo3Channels()
resize = Resize(height=img_height, width=img_width)
dataset_size = dataset.get_dataset_size()
print("Number of images in the dataset:\t{:>6}".format(dataset_size))
predict_generator = dataset.generate(
batch_size=config.PREDICT_BATCH_SIZE,
shuffle=True,
transformations=[convert_to_3_channels, resize],
label_encoder=None,
returns={
'processed_images', 'filenames', 'inverse_transform',
'original_images', 'original_labels'
},
keep_images_without_gt=False)
count = 0
while True and count < dataset_size:
batch_images, batch_filenames, batch_inverse_transforms, batch_original_images, batch_original_labels = next(
predict_generator)
i = 0
print("Image:", batch_filenames[i])
print()
print("Ground truth boxes:\n")
print(np.array(batch_original_labels[i]))
y_pred = model.predict(batch_images)
y_pred_decoded = decode_detections(y_pred,
confidence_thresh=0.5,
iou_threshold=0.4,
top_k=200,
normalize_coords=normalize_coords,
img_height=img_height,
img_width=img_width)
y_pred_decoded_inv = apply_inverse_transforms(
y_pred_decoded, batch_inverse_transforms)
np.set_printoptions(precision=2, suppress=True, linewidth=90)
print("Predicted boxes:\n")
print(' class conf xmin ymin xmax ymax')
print(y_pred_decoded_inv[i])
# cv2.imshow('original image',batch_original_images[i])
# cv2.waitKey(800)
# cv2.destroyAllWindows()
colors = plt.cm.hsv(np.linspace(0, 1, n_classes + 1)).tolist()
plt.figure(figsize=(15, 8))
plt.imshow(batch_original_images[i])
current_axis = plt.gca()
len_orig = 0
for box in batch_original_labels[i]:
len_orig += 1
xmin = box[1]
ymin = box[2]
xmax = box[3]
ymax = box[4]
label = '{}'.format(classes[int(box[0])])
current_axis.add_patch(
plt.Rectangle((xmin, ymin),
xmax - xmin,
ymax - ymin,
color='green',
fill=False,
linewidth=2))
current_axis.text(xmin,
ymin,
label,
size='x-large',
color='white',
bbox={
'facecolor': 'green',
'alpha': 1.0
})
len_found = 0
for box in y_pred_decoded_inv[i]:
len_found += 1
xmin = box[2]
ymin = box[3]
xmax = box[4]
ymax = box[5]
color = colors[int(box[0])]
label = '{}: {:.2f}'.format(classes[int(box[0])], box[1])
current_axis.add_patch(
plt.Rectangle((xmin, ymin),
xmax - xmin,
ymax - ymin,
color=color,
fill=False,
linewidth=2))
current_axis.text(xmin,
ymin,
label,
size='x-large',
color='white',
bbox={
'facecolor': color,
'alpha': 1.0
})
print('Number of original boxes : {}'.format(len_orig))
print('Number of found boxes : {}'.format(len_found))
plt.show()
count += 1
def predict_all_to_json(out_file,
model,
img_height,
img_width,
classes_to_cats,
data_generator,
batch_size,
data_generator_mode='resize',
model_mode='training',
confidence_thresh=0.01,
iou_threshold=0.45,
top_k=200,
pred_coords='centroids',
normalize_coords=True):
'''
Runs detection predictions over the whole dataset given a model and saves them in a JSON file
in the MS COCO detection results format.
Arguments:
out_file (str): The file name (full path) under which to save the results JSON file.
model (Keras model): A Keras SSD model object.
img_height (int): The input image height for the model.
img_width (int): The input image width for the model.
classes_to_cats (dict): A dictionary that maps the consecutive class IDs predicted by the model
to the non-consecutive original MS COCO category IDs.
data_generator (DataGenerator): A `DataGenerator` object with the evaluation dataset.
batch_size (int): The batch size for the evaluation.
data_generator_mode (str, optional): Either of 'resize' or 'pad'. If 'resize', the input images will
be resized (i.e. warped) to `(img_height, img_width)`. This mode does not preserve the aspect ratios of the images.
If 'pad', the input images will be first padded so that they have the aspect ratio defined by `img_height`
and `img_width` and then resized to `(img_height, img_width)`. This mode preserves the aspect ratios of the images.
model_mode (str, optional): The mode in which the model was created, i.e. 'training', 'inference' or 'inference_fast'.
This is needed in order to know whether the model output is already decoded or still needs to be decoded. Refer to
the model documentation for the meaning of the individual modes.
confidence_thresh (float, optional): A float in [0,1), the minimum classification confidence in a specific
positive class in order to be considered for the non-maximum suppression stage for the respective class.
A lower value will result in a larger part of the selection process being done by the non-maximum suppression
stage, while a larger value will result in a larger part of the selection process happening in the confidence
thresholding stage.
iou_threshold (float, optional): A float in [0,1]. All boxes with a Jaccard similarity of greater than `iou_threshold`
with a locally maximal box will be removed from the set of predictions for a given class, where 'maximal' refers
to the box score.
top_k (int, optional): The number of highest scoring predictions to be kept for each batch item after the
non-maximum suppression stage. Defaults to 200, following the paper.
input_coords (str, optional): The box coordinate format that the model outputs. Can be either 'centroids'
for the format `(cx, cy, w, h)` (box center coordinates, width, and height), 'minmax' for the format
`(xmin, xmax, ymin, ymax)`, or 'corners' for the format `(xmin, ymin, xmax, ymax)`.
normalize_coords (bool, optional): Set to `True` if the model outputs relative coordinates (i.e. coordinates in [0,1])
and you wish to transform these relative coordinates back to absolute coordinates. If the model outputs
relative coordinates, but you do not want to convert them back to absolute coordinates, set this to `False`.
Do not set this to `True` if the model already outputs absolute coordinates, as that would result in incorrect
coordinates. Requires `img_height` and `img_width` if set to `True`.
Returns:
None.
'''
convert_to_3_channels = ConvertTo3Channels()
resize = Resize(height=img_height,width=img_width)
if data_generator_mode == 'resize':
transformations = [convert_to_3_channels,
resize]
elif data_generator_mode == 'pad':
random_pad = RandomPadFixedAR(patch_aspect_ratio=img_width/img_height, clip_boxes=False)
transformations = [convert_to_3_channels,
random_pad,
resize]
else:
raise ValueError("Unexpected argument value: `data_generator_mode` can be either of 'resize' or 'pad', but received '{}'.".format(data_generator_mode))
# Set the generator parameters.
generator = data_generator.generate(batch_size=batch_size,
shuffle=False,
transformations=transformations,
label_encoder=None,
returns={'processed_images',
'image_ids',
'inverse_transform'},
keep_images_without_gt=True)
# Put the results in this list.
results = []
# Compute the number of batches to iterate over the entire dataset.
n_images = data_generator.get_dataset_size()
print("Number of images in the evaluation dataset: {}".format(n_images))
n_batches = int(ceil(n_images / batch_size))
# Loop over all batches.
tr = trange(n_batches, file=sys.stdout)
tr.set_description('Producing results file')
for i in tr:
# Generate batch.
batch_X, batch_image_ids, batch_inverse_transforms = next(generator)
# Predict.
y_pred = model.predict(batch_X)
# If the model was created in 'training' mode, the raw predictions need to
# be decoded and filtered, otherwise that's already taken care of.
if model_mode == 'training':
# Decode.
y_pred = decode_detections(y_pred,
confidence_thresh=confidence_thresh,
iou_threshold=iou_threshold,
top_k=top_k,
input_coords=pred_coords,
normalize_coords=normalize_coords,
img_height=img_height,
img_width=img_width)
else:
# Filter out the all-zeros dummy elements of `y_pred`.
y_pred_filtered = []
for i in range(len(y_pred)):
y_pred_filtered.append(y_pred[i][y_pred[i,:,0] != 0])
y_pred = y_pred_filtered
# Convert the predicted box coordinates for the original images.
y_pred = apply_inverse_transforms(y_pred, batch_inverse_transforms)
# Convert each predicted box into the results format.
for k, batch_item in enumerate(y_pred):
for box in batch_item:
class_id = box[0]
# Transform the consecutive class IDs back to the original COCO category IDs.
cat_id = classes_to_cats[class_id]
# Round the box coordinates to reduce the JSON file size.
xmin = float(round(box[2], 1))
ymin = float(round(box[3], 1))
xmax = float(round(box[4], 1))
ymax = float(round(box[5], 1))
width = xmax - xmin
height = ymax - ymin
bbox = [xmin, ymin, width, height]
result = {}
result['image_id'] = batch_image_ids[k]
result['category_id'] = cat_id
result['score'] = float(round(box[1], 3))
result['bbox'] = bbox
results.append(result)
with open(out_file, 'w') as f:
json.dump(results, f)
print("Prediction results saved in '{}'".format(out_file))
#print(np.array(batch_original_labels[i]))
#y_pred = model.predict(batch_images)
start = time.time()
y_pred = model.predict([batch_images, masks])
elapsed = (time.time() - start) * 1000
times.append(elapsed)
#print("Time: ",time.time()-start)
# Perform confidence thresholding.
y_pred_thresh = [
y_pred[k][y_pred[k, :, 1] > confidence_threshold]
for k in range(y_pred.shape[0])
]
# Convert the predictions for the original image.
y_pred_thresh_inv = apply_inverse_transforms(y_pred_thresh,
batch_inverse_transforms)
#np.set_printoptions(precision=2, suppress=True, linewidth=90)
#print("Predicted boxes:\n")
#print(' class conf xmin ymin xmax ymax')
#print(y_pred_thresh_inv[i])
# Display the image and draw the predicted boxes onto it.
#current_axis = plt.gca()
gts = []
dets = []
for box in batch_original_labels[i]:
xmin = box[1]
ymin = box[2]
xmax = box[3]
def predict_on_dataset(self,
img_height,
img_width,
batch_size,
data_generator_mode='resize',
decoding_confidence_thresh=0.01,
decoding_iou_threshold=0.45,
decoding_top_k=200,
decoding_pred_coords='centroids',
decoding_normalize_coords=True,
decoding_border_pixels='include',
round_confidences=False,
verbose=True,
ret=False):
class_id_pred = self.pred_format['class_id']
conf_pred = self.pred_format['conf']
xmin_pred = self.pred_format['xmin']
ymin_pred = self.pred_format['ymin']
xmax_pred = self.pred_format['xmax']
ymax_pred = self.pred_format['ymax']
convert_to_3_channels = ConvertTo3Channels()
resize = Resize(height=img_height,
width=img_width,
labels_format=self.gt_format)
if data_generator_mode == 'resize':
transformations = [convert_to_3_channels, resize]
elif data_generator_mode == 'pad':
random_pad = RandomPadFixedAR(patch_aspect_ratio=img_width /
img_height,
labels_format=self.gt_format)
transformations = [convert_to_3_channels, random_pad, resize]
else:
raise ValueError(
"`data_generator_mode` can be either of 'resize' or 'pad', but received '{}'."
.format(data_generator_mode))
# Set the generator parameters.
generator = self.data_generator.generate(
batch_size=batch_size,
shuffle=False,
transformations=transformations,
label_encoder=None,
returns={
'processed_images', 'image_ids', 'evaluation-neutral',
'inverse_transform', 'original_labels'
},
keep_images_without_gt=True,
degenerate_box_handling='remove')
if self.data_generator.image_ids is None:
self.data_generator.image_ids = list(
range(self.data_generator.get_dataset_size()))
#############################################################################################
# Predict over all batches of the dataset and store the predictions.
#############################################################################################
# We have to generate a separate results list for each class.
results = [list() for _ in range(self.n_classes + 1)]
# Create a dictionary that maps image IDs to ground truth annotations.
# We'll need it below.
image_ids_to_labels = {}
# Compute the number of batches to iterate over the entire dataset.
n_images = self.data_generator.get_dataset_size()
n_batches = int(floor(n_images / batch_size)) - 1
if verbose:
print("Number of images in the evaluation dataset: {}".format(
n_images))
print()
tr = trange(n_batches, file=sys.stdout)
tr.set_description('Producing predictions batch-wise')
else:
tr = range(n_batches)
# Loop over all batches.
for j in tr:
# Generate batch.
batch_X, batch_image_ids, batch_eval_neutral, batch_inverse_transforms, batch_orig_labels = next(
generator)
# Predict.
y_pred = self.model.predict(batch_X)
if self.model_mode == 'training':
# Decode.
y_pred = decode_detections(
y_pred,
confidence_thresh=decoding_confidence_thresh,
iou_threshold=decoding_iou_threshold,
top_k=decoding_top_k,
input_coords=decoding_pred_coords,
normalize_coords=decoding_normalize_coords,
img_height=img_height,
img_width=img_width,
border_pixels=decoding_border_pixels)
else:
# Filter out the all-zeros dummy elements of `y_pred`.
y_pred_filtered = []
for i in range(len(y_pred)):
y_pred_filtered.append(y_pred[i][y_pred[i, :, 0] != 0])
y_pred = y_pred_filtered
# Convert the predicted box coordinates for the original images.
y_pred = apply_inverse_transforms(y_pred, batch_inverse_transforms)
# Iterate over all batch items.
for k, batch_item in enumerate(y_pred):
image_id = batch_image_ids[k]
for box in batch_item:
class_id = int(box[class_id_pred])
# Round the box coordinates to reduce the required memory.
if round_confidences:
confidence = round(box[conf_pred], round_confidences)
else:
confidence = box[conf_pred]
xmin = round(box[xmin_pred], 1)
ymin = round(box[ymin_pred], 1)
xmax = round(box[xmax_pred], 1)
ymax = round(box[ymax_pred], 1)
prediction = (image_id, confidence, xmin, ymin, xmax, ymax)
# Append the predicted box to the results list for its class.
results[class_id].append(prediction)
self.prediction_results = results
if ret:
return results
def _main_(args):
print('Hello World! This is {:s}'.format(args.desc))
# config_path = args.conf
# with open(config_path) as config_buffer:
# config = json.loads(config_buffer.read())
#############################################################
# Set model parameters
#############################################################
img_height = 300 # Height of the model input images
img_width = 300 # Width of the model input images
img_channels = 3 # Number of color channels of the model input images
mean_color = [123, 117, 104] # The per-channel mean of the images in the dataset. Do not change this value if you're using any of the pre-trained weights.
swap_channels = [2, 1, 0] # The color channel order in the original SSD is BGR, so we'll have the model reverse the color channel order of the input images.
n_classes = 20 # Number of positive classes, e.g. 20 for Pascal VOC, 80 for MS COCO
scales_pascal = [0.1, 0.2, 0.37, 0.54, 0.71, 0.88, 1.05] # The anchor box scaling factors used in the original SSD300 for the Pascal VOC datasets
scales_coco = [0.07, 0.15, 0.33, 0.51, 0.69, 0.87, 1.05] # The anchor box scaling factors used in the original SSD300 for the MS COCO datasets
scales = scales_pascal
aspect_ratios = [[1.0, 2.0, 0.5],
[1.0, 2.0, 0.5, 3.0, 1.0 / 3.0],
[1.0, 2.0, 0.5, 3.0, 1.0 / 3.0],
[1.0, 2.0, 0.5, 3.0, 1.0 / 3.0],
[1.0, 2.0, 0.5],
[1.0, 2.0, 0.5]] # The anchor box aspect ratios used in the original SSD300; the order matters
two_boxes_for_ar1 = True
steps = [8, 16, 32, 64, 100, 300] # The space between two adjacent anchor box center points for each predictor layer.
offsets = [0.5, 0.5, 0.5, 0.5, 0.5, 0.5] # The offsets of the first anchor box center points from the top and left borders of the image as a fraction of the step size for each predictor layer.
clip_boxes = False # Whether or not to clip the anchor boxes to lie entirely within the image boundaries
variances = [0.1, 0.1, 0.2, 0.2] # The variances by which the encoded target coordinates are divided as in the original implementation
normalize_coords = True
#############################################################
# Create the model
#############################################################
# 1: Build the Keras model.
model = ssd_300(image_size=(img_height, img_width, img_channels),
n_classes=n_classes,
mode='training',
l2_regularization=0.0005,
scales=scales,
aspect_ratios_per_layer=aspect_ratios,
two_boxes_for_ar1=two_boxes_for_ar1,
steps=steps,
offsets=offsets,
clip_boxes=clip_boxes,
variances=variances,
normalize_coords=normalize_coords,
subtract_mean=mean_color,
swap_channels=swap_channels)
# 2: Load some weights into the model.
# 3: Instantiate an optimizer and the SSD loss function and compile the model.
adam = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0)
ssd_loss = SSDLoss(neg_pos_ratio=3, alpha=1.0)
model.compile(optimizer=adam, loss=ssd_loss.compute_loss)
#############################################################
# Prepare the data
#############################################################
# 1: Instantiate two `DataGenerator` objects: One for training, one for validation.
train_dataset = DataGenerator(load_images_into_memory=False, hdf5_dataset_path=None)
val_dataset = DataGenerator(load_images_into_memory=False, hdf5_dataset_path=None)
# 2: Parse the image and label lists for the training and validation datasets. This can take a while.
VOC_2007_images_dir = '/home/minhnc-lab/WORKSPACES/AI/data/VOC/VOCtrainval_06-Nov-2007/VOCdevkit/VOC2007/JPEGImages'
VOC_2007_annotations_dir = '/home/minhnc-lab/WORKSPACES/AI/data/VOC/VOCtrainval_06-Nov-2007/VOCdevkit/VOC2007/Annotations'
VOC_2007_train_image_set_filename = '/home/minhnc-lab/WORKSPACES/AI/data/VOC/VOCtrainval_06-Nov-2007/VOCdevkit/VOC2007/ImageSets/Main/train.txt'
VOC_2007_val_image_set_filename = '/home/minhnc-lab/WORKSPACES/AI/data/VOC/VOCtrainval_06-Nov-2007/VOCdevkit/VOC2007/ImageSets/Main/val.txt'
# VOC_2007_trainval_image_set_filename = '/home/minhnc-lab/WORKSPACES/AI/data/VOC/VOCtrainval_06-Nov-2007/VOCdevkit/VOC2007/ImageSets/Main/trainval.txt'
# VOC_2007_test_image_set_filename = '/home/minhnc-lab/WORKSPACES/AI/data/VOC/VOCtest_06-Nov-2007/VOCdevkit/VOC2007/ImageSets/Main/test.txt'
classes = ['background',
'aeroplane', 'bicycle', 'bird', 'boat',
'bottle', 'bus', 'car', 'cat',
'chair', 'cow', 'diningtable', 'dog',
'horse', 'motorbike', 'person', 'pottedplant',
'sheep', 'sofa', 'train', 'tvmonitor']
train_dataset.parse_xml(images_dirs=[VOC_2007_images_dir],
image_set_filenames=[VOC_2007_train_image_set_filename],
annotations_dirs=[VOC_2007_annotations_dir],
classes=classes,
include_classes='all',
exclude_truncated=False,
exclude_difficult=False,
ret=False)
val_dataset.parse_xml(images_dirs=[VOC_2007_images_dir],
image_set_filenames=[VOC_2007_val_image_set_filename],
annotations_dirs=[VOC_2007_annotations_dir],
classes=classes,
include_classes='all',
exclude_truncated=False,
exclude_difficult=True,
ret=False)
train_dataset.create_hdf5_dataset(file_path='dataset_pascal_voc_07+12_trainval.h5',
resize=False,
variable_image_size=True,
verbose=True)
val_dataset.create_hdf5_dataset(file_path='dataset_pascal_voc_07_test.h5',
resize=False,
variable_image_size=True,
verbose=True)
# 3: Set the batch size.
batch_size = 8 # Change the batch size if you like, or if you run into GPU memory issues.
# 4: Set the image transformations for pre-processing and data augmentation options.
ssd_data_augmentation = SSDDataAugmentation(img_height=img_height,
img_width=img_width,
background=mean_color)
convert_to_3_channels = ConvertTo3Channels()
resize = Resize(height=img_height, width=img_width)
# 5: Instantiate an encoder that can encode ground truth labels into the format needed by the SSD loss function.
predictor_sizes = [model.get_layer('conv4_3_norm_mbox_conf').output_shape[1:3],
model.get_layer('fc7_mbox_conf').output_shape[1:3],
model.get_layer('conv6_2_mbox_conf').output_shape[1:3],
model.get_layer('conv7_2_mbox_conf').output_shape[1:3],
model.get_layer('conv8_2_mbox_conf').output_shape[1:3],
model.get_layer('conv9_2_mbox_conf').output_shape[1:3]]
ssd_input_encoder = SSDInputEncoder(img_height=img_height,
img_width=img_width,
n_classes=n_classes,
predictor_sizes=predictor_sizes,
scales=scales,
aspect_ratios_per_layer=aspect_ratios,
two_boxes_for_ar1=two_boxes_for_ar1,
steps=steps,
offsets=offsets,
clip_boxes=clip_boxes,
variances=variances,
matching_type='multi',
pos_iou_threshold=0.5,
neg_iou_limit=0.5,
normalize_coords=normalize_coords)
# 6: Create the generator handles that will be passed to Keras' `fit_generator()` function.
train_generator = train_dataset.generate(batch_size=batch_size,
shuffle=True,
transformations=[ssd_data_augmentation],
label_encoder=ssd_input_encoder,
returns={'processed_images',
'encoded_labels'},
keep_images_without_gt=False)
val_generator = val_dataset.generate(batch_size=batch_size,
shuffle=False,
transformations=[convert_to_3_channels,
resize],
label_encoder=ssd_input_encoder,
returns={'processed_images',
'encoded_labels'},
keep_images_without_gt=False)
# Get the number of samples in the training and validations datasets.
train_dataset_size = train_dataset.get_dataset_size()
val_dataset_size = val_dataset.get_dataset_size()
print("Number of images in the training dataset:\t{:>6}".format(train_dataset_size))
print("Number of images in the validation dataset:\t{:>6}".format(val_dataset_size))
#############################################################
# Kick off the training
#############################################################
# Define model callbacks.
model_checkpoint = ModelCheckpoint(
filepath='ssd300_pascal_07+12_epoch-{epoch:02d}_loss-{loss:.4f}_val_loss-{val_loss:.4f}.h5',
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='auto',
period=1)
csv_logger = CSVLogger(filename='ssd300_pascal_07+12_training_log.csv',
separator=',',
append=True)
learning_rate_scheduler = LearningRateScheduler(schedule=lr_schedule,
verbose=1)
terminate_on_nan = TerminateOnNaN()
callbacks = [model_checkpoint,
csv_logger,
learning_rate_scheduler,
terminate_on_nan]
# Train
initial_epoch = 0
final_epoch = 120
steps_per_epoch = 1000
history = model.fit_generator(generator=train_generator,
steps_per_epoch=steps_per_epoch,
epochs=final_epoch,
callbacks=callbacks,
validation_data=val_generator,
validation_steps=ceil(val_dataset_size / batch_size),
initial_epoch=initial_epoch)
#############################################################
# Run the evaluation
#############################################################
# 1: Set the generator for the predictions.
predict_generator = val_dataset.generate(batch_size=1,
shuffle=True,
transformations=[convert_to_3_channels,
resize],
label_encoder=None,
returns={'processed_images',
'filenames',
'inverse_transform',
'original_images',
'original_labels'},
keep_images_without_gt=False)
# 2: Generate samples.
batch_images, batch_filenames, batch_inverse_transforms, batch_original_images, batch_original_labels = next(
predict_generator)
i = 0 # Which batch item to look at
print("Image:", batch_filenames[i])
print()
print("Ground truth boxes:\n")
print(np.array(batch_original_labels[i]))
# 3: Make predictions.
y_pred = model.predict(batch_images)
# 4: Decode the raw predictions in `y_pred`.
y_pred_decoded = decode_detections(y_pred,
confidence_thresh=0.5,
iou_threshold=0.4,
top_k=200,
normalize_coords=normalize_coords,
img_height=img_height,
img_width=img_width)
# 5: Convert the predictions for the original image.
y_pred_decoded_inv = apply_inverse_transforms(y_pred_decoded, batch_inverse_transforms)
np.set_printoptions(precision=2, suppress=True, linewidth=90)
print("Predicted boxes:\n")
print(' class conf xmin ymin xmax ymax')
print(y_pred_decoded_inv[i])
# 6: Draw the predicted boxes onto the image
# Set the colors for the bounding boxes
colors = plt.cm.hsv(np.linspace(0, 1, n_classes + 1)).tolist()
classes = ['background',
'aeroplane', 'bicycle', 'bird', 'boat',
'bottle', 'bus', 'car', 'cat',
'chair', 'cow', 'diningtable', 'dog',
'horse', 'motorbike', 'person', 'pottedplant',
'sheep', 'sofa', 'train', 'tvmonitor']
plt.figure(figsize=(20, 12))
plt.imshow(batch_original_images[i])
current_axis = plt.gca()
for box in batch_original_labels[i]:
xmin = box[1]
ymin = box[2]
xmax = box[3]
ymax = box[4]
label = '{}'.format(classes[int(box[0])])
current_axis.add_patch(
plt.Rectangle((xmin, ymin), xmax - xmin, ymax - ymin, color='green', fill=False, linewidth=2))
current_axis.text(xmin, ymin, label, size='x-large', color='white', bbox={'facecolor': 'green', 'alpha': 1.0})
for box in y_pred_decoded_inv[i]:
xmin = box[2]
ymin = box[3]
xmax = box[4]
ymax = box[5]
color = colors[int(box[0])]
label = '{}: {:.2f}'.format(classes[int(box[0])], box[1])
current_axis.add_patch(
plt.Rectangle((xmin, ymin), xmax - xmin, ymax - ymin, color=color, fill=False, linewidth=2))
current_axis.text(xmin, ymin, label, size='x-large', color='white', bbox={'facecolor': color, 'alpha': 1.0})
# weights_conv = model.get_layer('conv1_1').get_weights()[0] # weight each layer
# 4: Decode the raw predictions in `y_pred`.
y_pred_decoded = decode_detections(y_pred,
confidence_thresh=0.2,
iou_threshold=0.2,
top_k=200,
normalize_coords=normalize_coords,
img_height=img_height,
img_width=img_width)
# 5: Convert the predictions for the original image.
y_pred_decoded_inv = apply_inverse_transforms(y_pred_decoded, batch_inverse_transforms) # แปลงภาพกลับเป็นแบบ เดิม
print(y_pred_decoded_inv)
np.set_printoptions(precision=2, suppress=True, linewidth=90)
print("Predicted boxes:\n")
print(' class conf xmin ymin xmax ymax')
print(y_pred_decoded_inv[i])
# 5: Draw the predicted boxes onto the image
# Set the colors for the bounding boxes
colors = plt.cm.hsv(np.linspace(0, 1, n_classes+1)).tolist()
'''
classes = ['background',
'aeroplane', 'bicycle', 'bird', 'boat',
'bottle', 'bus', 'car', 'cat',
'chair', 'cow', 'diningtable', 'dog',
def predict_on_dataset(self,
img_height,
img_width,
batch_size,
data_generator_mode='resize',
decoding_confidence_thresh=0.01,
decoding_iou_threshold=0.45,
decoding_top_k=200,
decoding_pred_coords='centroids',
decoding_normalize_coords=True,
decoding_border_pixels='include',
round_confidences=False,
verbose=True,
ret=False):
'''
Runs predictions for the given model over the entire dataset given by `data_generator`.
Arguments:
img_height (int): The input image height for the model.
img_width (int): The input image width for the model.
batch_size (int): The batch size for the evaluation.
data_generator_mode (str, optional): Either of 'resize' and 'pad'. If 'resize', the input images will
be resized (i.e. warped) to `(img_height, img_width)`. This mode does not preserve the aspect ratios of the images.
If 'pad', the input images will be first padded so that they have the aspect ratio defined by `img_height`
and `img_width` and then resized to `(img_height, img_width)`. This mode preserves the aspect ratios of the images.
decoding_confidence_thresh (float, optional): Only relevant if the model is in 'training' mode.
A float in [0,1), the minimum classification confidence in a specific positive class in order to be considered
for the non-maximum suppression stage for the respective class. A lower value will result in a larger part of the
selection process being done by the non-maximum suppression stage, while a larger value will result in a larger
part of the selection process happening in the confidence thresholding stage.
decoding_iou_threshold (float, optional): Only relevant if the model is in 'training' mode. A float in [0,1].
All boxes with a Jaccard similarity of greater than `iou_threshold` with a locally maximal box will be removed
from the set of predictions for a given class, where 'maximal' refers to the box score.
decoding_top_k (int, optional): Only relevant if the model is in 'training' mode. The number of highest scoring
predictions to be kept for each batch item after the non-maximum suppression stage.
decoding_input_coords (str, optional): Only relevant if the model is in 'training' mode. The box coordinate format
that the model outputs. Can be either 'centroids' for the format `(cx, cy, w, h)` (box center coordinates, width, and height),
'minmax' for the format `(xmin, xmax, ymin, ymax)`, or 'corners' for the format `(xmin, ymin, xmax, ymax)`.
decoding_normalize_coords (bool, optional): Only relevant if the model is in 'training' mode. Set to `True` if the model
outputs relative coordinates. Do not set this to `True` if the model already outputs absolute coordinates,
as that would result in incorrect coordinates.
round_confidences (int, optional): `False` or an integer that is the number of decimals that the prediction
confidences will be rounded to. If `False`, the confidences will not be rounded.
verbose (bool, optional): If `True`, will print out the progress during runtime.
ret (bool, optional): If `True`, returns the predictions.
Returns:
None by default. Optionally, a nested list containing the predictions for each class.
'''
class_id_pred = self.pred_format['class_id']
conf_pred = self.pred_format['conf']
xmin_pred = self.pred_format['xmin']
ymin_pred = self.pred_format['ymin']
xmax_pred = self.pred_format['xmax']
ymax_pred = self.pred_format['ymax']
#############################################################################################
# Configure the data generator for the evaluation.
#############################################################################################
convert_to_3_channels = ConvertTo3Channels()
resize = Resize(height=img_height,width=img_width, labels_format=self.gt_format)
if data_generator_mode == 'resize':
transformations = [convert_to_3_channels,
resize]
elif data_generator_mode == 'pad':
random_pad = RandomPadFixedAR(patch_aspect_ratio=img_width/img_height, labels_format=self.gt_format)
transformations = [convert_to_3_channels,
random_pad,
resize]
else:
raise ValueError("`data_generator_mode` can be either of 'resize' or 'pad', but received '{}'.".format(data_generator_mode))
# Set the generator parameters.
generator = self.data_generator.generate(batch_size=batch_size,
shuffle=False,
transformations=transformations,
label_encoder=None,
returns={'processed_images',
'image_ids',
'evaluation-neutral',
'inverse_transform',
'original_labels'},
keep_images_without_gt=True,
degenerate_box_handling='remove')
if self.data_generator.image_ids is None:
self.data_generator.image_ids = list(range(self.data_generator.get_dataset_size()))
#############################################################################################
# Predict over all batches of the dataset and store the predictions.
#############################################################################################
# We have to generate a separate results list for each class.
results = [list() for _ in range(self.n_classes + 1)]
# Create a dictionary that maps image IDs to ground truth annotations.
# We'll need it below.
image_ids_to_labels = {}
# Compute the number of batches to iterate over the entire dataset.
n_images = self.data_generator.get_dataset_size()
n_batches = int(ceil(n_images / batch_size))
if verbose:
print("Number of images in the evaluation dataset: {}".format(n_images))
print()
tr = trange(n_batches, file=sys.stdout)
tr.set_description('Producing predictions batch-wise')
else:
tr = range(n_batches)
for j in tr:
# Generate batch.
batch_X, batch_image_ids, batch_eval_neutral, batch_inverse_transforms, batch_orig_labels = next(generator)
# Predict.
y_pred = self.model.predict(batch_X)
# If the model was created in 'training' mode, the raw predictions need to
# be decoded and filtered, otherwise that's already taken care of.
if self.model_mode == 'training':
# Decode.
y_pred = decode_detections(y_pred,
confidence_thresh=decoding_confidence_thresh,
iou_threshold=decoding_iou_threshold,
top_k=decoding_top_k,
input_coords=decoding_pred_coords,
normalize_coords=decoding_normalize_coords,
img_height=img_height,
img_width=img_width,
border_pixels=decoding_border_pixels)
else:
# Filter out the all-zeros dummy elements of `y_pred`.
y_pred_filtered = []
for i in range(len(y_pred)):
y_pred_filtered.append(y_pred[i][y_pred[i,:,0] != 0])
y_pred = y_pred_filtered
# Convert the predicted box coordinates for the original images.
y_pred = apply_inverse_transforms(y_pred, batch_inverse_transforms)
# Iterate over all batch items.
for k, batch_item in enumerate(y_pred):
image_id = batch_image_ids[k]
path='/data/deeplearn/SWEIPENet/dataset/Detections/detection'+ str(self.modelindex)
if not os.path.exists(path):
os.mkdir(path)
txtpath = os.path.join(path, image_id + '.txt')
if not os.path.exists(txtpath):
os.mknod(txtpath)
file_fid = open(txtpath, 'w')
for box in batch_item:
class_id = int(box[class_id_pred])
# Round the box coordinates to reduce the required memory.
if round_confidences:
confidence = round(box[conf_pred], round_confidences)
else:
confidence = box[conf_pred]
xmin = round(box[xmin_pred], 1)
ymin = round(box[ymin_pred], 1)
xmax = round(box[xmax_pred], 1)
ymax = round(box[ymax_pred], 1)
prediction = (image_id, confidence, xmin, ymin, xmax, ymax)
# write detections of each image into Detections/imname.txt
if class_id == 1:
class_name = 'seacucumber'
if class_id == 2:
class_name = 'seaurchin'
if class_id == 3:
class_name = 'scallop'
boxstr = class_name + ' ' + str(confidence)+ ' ' + str(xmin) + ' ' + str(ymin) + ' ' + str(xmax) + ' ' + str(ymax)
file_fid.write(boxstr + '\n')
results[class_id].append(prediction)
file_fid.close()
self.prediction_results = results
if ret:
return results
def predicting(images,
image_path,
labels_output_format=('class_id', 'xmin', 'ymin', 'xmax',
'ymax')):
labels_format = {
'class_id': labels_output_format.index('class_id'),
'xmin': labels_output_format.index('xmin'),
'ymin': labels_output_format.index('ymin'),
'xmax': labels_output_format.index('xmax'),
'ymax': labels_output_format.index('ymax')
}
convert_to_3_channels = ConvertTo3Channels()
resize = Resize(height=img_height, width=img_width)
generate_pre = Data_Generator(image=images,
image_path=image_path,
labels_format=labels_format)
predict_generator = generate_pre.generate(
batch_size=1,
transformations=[convert_to_3_channels, resize],
label_encoder=None,
returns={
'processed_images', 'filenames', 'inverse_transform',
'original_images', 'original_labels'
},
keep_images_without_gt=False)
batch_images, batch_filenames, batch_inverse_transforms, batch_original_images, batch_original_labels = next(
predict_generator)
i = 0
print("Image:", "????")
global graph
with graph.as_default():
y_pred = model.predict(batch_images)
y_pred_decoded = decode_detections(y_pred,
confidence_thresh=0.25,
iou_threshold=0.4,
top_k=200,
normalize_coords=normalize_coords,
img_height=img_height,
img_width=img_width)
print(y_pred_decoded)
y_pred_decoded_inv = apply_inverse_transforms(y_pred_decoded,
batch_inverse_transforms)
# print(y_pred_decoded_inv)
np.set_printoptions(precision=2, suppress=True, linewidth=90)
# print("Predicted boxes:\n")
# print(' class conf xmin ymin xmax ymax')
# print(y_pred_decoded_inv[i])
# Set the colors for the bounding boxes
# plt.figure(figsize=(20, 12))
# plt.imshow(batch_original_images[i])
# colors = plt.cm.hsv(np.linspace(0, 1, n_classes + 1)).tolist()
# current_axis = plt.gca()
#
# for box in y_pred_decoded_inv[i]:
# xmin = box[2]
# ymin = box[3]
# xmax = box[4]
# ymax = box[5]
# # color = colors[int(box[0])]
# # plt.Rectangle((xmin, ymin), xmax - xmin, ymax - ymin, color=color, fill=False, linewidth=2)
# color = colors[int(box[0])]
# label = '{}: {:.2f}'.format(classes[int(box[0])], box[1])
# current_axis.add_patch(
# plt.Rectangle((xmin, ymin), xmax - xmin, ymax - ymin, color=color, fill=False, linewidth=2))
# current_axis.text(xmin, ymin, label, size='x-large', color='white', bbox={'facecolor': color, 'alpha': 1.0})
#
# # plt.show()
# plt.xticks([])
# plt.yticks([])
# plt.savefig(self.image_path)
# K.clear_session()
# gc.collect()
return y_pred_decoded_inv
def run(train_dir, valid_dir, set_dir, model_dir):
# train_dir = arguments.train_dir
# valid_dir = arguments.valid_dir
train_dataset_dir = train_dir
train_annot_dir = train_dir + '/annot/'
train_set = train_dir + '/img_set.txt'
valid_dataset_dir = valid_dir
valid_annot_dir = valid_dir + '/annot/'
valid_set = valid_dir + '/valid_set.txt'
# Set Training and Validation dataset paths
batch_size = 16
print('Using batch size of: {}'.format(batch_size))
#model_path = 'COCO_512.h5'
model_path = model_dir
# model_path = 'saved_model.h5'
# Needs to know classes and order to map to integers
classes = ['background', 'car', 'bus', 'truck']
# Set required parameters for training of SSD
img_height = 512
img_width = 512
img_channels = 3 # Colour image
mean_color = [123, 117, 104] # DO NOT CHANGE
swap_channels = [2, 1, 0] # Original SSD used BGR
n_classes = 3 # 80 for COCO
scales_coco = [0.04, 0.1, 0.26, 0.42, 0.58, 0.74, 0.9, 1.06]
scales = scales_coco
aspect_ratios = [[1.0, 2.0, 0.5], [1.0, 2.0, 0.5, 3.0, 1.0 / 3.0],
[1.0, 2.0, 0.5, 3.0, 1.0 / 3.0],
[1.0, 2.0, 0.5, 3.0, 1.0 / 3.0],
[1.0, 2.0, 0.5, 3.0, 1.0 / 3.0], [1.0, 2.0, 0.5],
[1.0, 2.0, 0.5]]
two_boxes_for_ar1 = True
steps = [8, 16, 32, 64, 128, 256, 512]
offsets = [0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5]
clip_boxes = False
variances = [0.1, 0.1, 0.2, 0.2]
normalize_coords = True
K.clear_session()
model = ssd_512(image_size=(img_height, img_width, img_channels),
n_classes=n_classes,
mode='training',
l2_regularization=0.0005,
scales=scales,
aspect_ratios_per_layer=aspect_ratios,
two_boxes_for_ar1=two_boxes_for_ar1,
steps=steps,
offsets=offsets,
clip_boxes=clip_boxes,
variances=variances,
normalize_coords=normalize_coords,
subtract_mean=mean_color,
swap_channels=swap_channels)
model.load_weights(model_path, by_name=True)
sgd = SGD(lr=0.001, momentum=0.9, decay=0.0, nesterov=False)
ssd_loss = SSDLoss(neg_pos_ratio=3, alpha=1.0)
model.compile(optimizer=sgd, loss=ssd_loss.compute_loss)
# model = load_model(model_path, custom_objects={'AnchorBoxes': AnchorBoxes,
# 'L2Normalization': L2Normalization,
# 'compute_loss': ssd_loss.compute_loss})
# Create Data Generators for train and valid sets
train_dataset = DataGenerator(load_images_into_memory=False,
hdf5_dataset_path=None)
valid_dataset = DataGenerator(load_images_into_memory=False,
hdf5_dataset_path=None)
train_dataset.parse_xml(images_dirs=[train_dataset_dir],
image_set_filenames=[train_set],
annotations_dirs=[train_annot_dir],
classes=classes,
include_classes='all',
exclude_truncated=False,
exclude_difficult=False,
ret=False)
valid_dataset.parse_xml(images_dirs=[valid_dataset_dir],
image_set_filenames=[valid_set],
annotations_dirs=[valid_annot_dir],
classes=classes,
include_classes='all',
exclude_truncated=False,
exclude_difficult=False,
ret=False)
# Will speed up trainig but requires more memory
# Can comment out to avoid memory requirements
'''
train_dataset.create_hdf5_dataset(file_path='dataset_pascal_voc_07+12_trainval.h5',
resize=False,
variable_image_size=True,
verbose=True)
valid_dataset.create_hdf5_dataset(file_path='dataset_pascal_voc_07_test.h5',
resize=False,
variable_image_size=True,
verbose=True)
'''
ssd_data_augmentation = SSDDataAugmentation(img_height=img_height,
img_width=img_width,
background=mean_color)
convert_to_3_channels = ConvertTo3Channels()
resize = Resize(height=img_height, width=img_width)
predictor_sizes = [
model.get_layer('conv4_3_norm_mbox_conf').output_shape[1:3],
model.get_layer('fc7_mbox_conf').output_shape[1:3],
model.get_layer('conv6_2_mbox_conf').output_shape[1:3],
model.get_layer('conv7_2_mbox_conf').output_shape[1:3],
model.get_layer('conv8_2_mbox_conf').output_shape[1:3],
model.get_layer('conv9_2_mbox_conf').output_shape[1:3],
model.get_layer('conv10_2_mbox_conf').output_shape[1:3]
]
ssd_input_encoder = SSDInputEncoder(img_height=img_height,
img_width=img_width,
n_classes=n_classes,
predictor_sizes=predictor_sizes,
scales=scales,
aspect_ratios_per_layer=aspect_ratios,
two_boxes_for_ar1=two_boxes_for_ar1,
steps=steps,
offsets=offsets,
clip_boxes=clip_boxes,
variances=variances,
matching_type='multi',
pos_iou_threshold=0.5,
neg_iou_limit=0.5,
normalize_coords=normalize_coords)
train_generator = train_dataset.generate(
batch_size=batch_size,
shuffle=True,
transformations=[ssd_data_augmentation],
label_encoder=ssd_input_encoder,
returns={'processed_images', 'encoded_labels'},
keep_images_without_gt=False)
val_generator = valid_dataset.generate(
batch_size=batch_size,
shuffle=False,
transformations=[convert_to_3_channels, resize],
label_encoder=ssd_input_encoder,
returns={'processed_images', 'encoded_labels'},
keep_images_without_gt=False)
# Get the number of samples in the training and validations datasets.
train_dataset_size = train_dataset.get_dataset_size()
valid_dataset_size = valid_dataset.get_dataset_size()
print("Number of images in the training dataset:\t{:>6}".format(
train_dataset_size))
print("Number of images in the validation dataset:\t{:>6}".format(
valid_dataset_size))
model_checkpoint = ModelCheckpoint(
filepath=
'ssd_epoch-{epoch:02d}_loss-{loss:.4f}_val_loss-{val_loss:.4f}.h5',
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='auto',
period=1)
#csv_logger = CSVLogger(filename='ssd512_training_log.csv',
# separator=',',
# append=True)
learning_rate_scheduler = LearningRateScheduler(schedule=lr_schedule,
verbose=1)
terminate_on_nan = TerminateOnNaN()
callbacks = [
model_checkpoint, csv_logger, learning_rate_scheduler, terminate_on_nan
]
#callbacks = [learning_rate_scheduler,
# terminate_on_nan]
initial_epoch = 0
final_epoch = 150 # 150
steps_per_epoch = math.ceil(119 /
batch_size) # ceil(num_samples/batch_size)
# Training
history = model.fit_generator(generator=train_generator,
steps_per_epoch=steps_per_epoch,
epochs=final_epoch,
callbacks=callbacks,
validation_data=val_generator,
validation_steps=math.ceil(
valid_dataset_size / batch_size),
initial_epoch=initial_epoch)
# Save final trained model
model.save('trained.h5')
# Make predictions
predict_generator = valid_dataset.generate(
batch_size=1,
shuffle=True,
transformations=[convert_to_3_channels, resize],
label_encoder=None,
returns={
'processed_images', 'filenames', 'inverse_transform',
'original_images', 'original_labels'
},
keep_images_without_gt=False)
batch_images, batch_filenames, batch_inverse_transforms, batch_original_images, batch_original_labels = next(
predict_generator)
i = 0 # Which batch item to look at
print("Image:", batch_filenames[i])
print()
print("Ground truth boxes:\n")
print(np.array(batch_original_labels[i]))
y_pred = model.predict(batch_images)
y_pred_decoded = decode_detections(y_pred,
confidence_thresh=0.2,
iou_threshold=0.4,
top_k=200,
normalize_coords=normalize_coords,
img_height=img_height,
img_width=img_width)
y_pred_decoded_inv = apply_inverse_transforms(y_pred_decoded,
batch_inverse_transforms)
np.set_printoptions(precision=2, suppress=True, linewidth=90)
print("Predicted boxes:\n")
print(' class conf xmin ymin xmax ymax')
print(y_pred_decoded_inv[i])
# Set the colors for the bounding boxes
colors = plt.cm.hsv(np.linspace(0, 1, n_classes + 1)).tolist()
# classes = ['background', 'car', 'bus', 'truck', 'motorbike'] # Already set at start
plt.figure(figsize=(20, 12))
plt.imshow(batch_original_images[i])
current_axis = plt.gca()
for box in batch_original_labels[i]:
xmin = box[1]
ymin = box[2]
xmax = box[3]
ymax = box[4]
label = '{}'.format(classes[int(box[0])])
current_axis.add_patch(
plt.Rectangle((xmin, ymin),
xmax - xmin,
ymax - ymin,
color='green',
fill=False,
linewidth=2))
current_axis.text(xmin,
ymin,
label,
size='x-large',
color='white',
bbox={
'facecolor': 'green',
'alpha': 1.0
})
for box in y_pred_decoded_inv[i]:
xmin = box[2]
ymin = box[3]
xmax = box[4]
ymax = box[5]
color = colors[int(box[0])]
label = '{}: {:.2f}'.format(classes[int(box[0])], box[1])
current_axis.add_patch(
plt.Rectangle((xmin, ymin),
xmax - xmin,
ymax - ymin,
color=color,
fill=False,
linewidth=2))
current_axis.text(xmin,
ymin,
label,
size='x-large',
color='white',
bbox={
'facecolor': color,
'alpha': 1.0
})
plt.show()
return
def predict_all_to_txt(model,
img_height,
img_width,
data_generator,
batch_size,
data_generator_mode='resize',
classes=[
'background', 'aeroplane', 'bicycle', 'bird',
'boat', 'bottle', 'bus', 'car', 'cat', 'chair',
'cow', 'diningtable', 'dog', 'horse', 'motorbike',
'person', 'pottedplant', 'sheep', 'sofa', 'train',
'tvmonitor'
],
out_file_prefix='comp3_det_test_',
model_mode='training',
confidence_thresh=0.01,
iou_threshold=0.45,
top_k=200,
pred_coords='centroids',
normalize_coords=True):
'''
Runs detection predictions over the whole dataset given a model and saves them in a text file
in the Pascal VOC detection results format, i.e. the format in which the Pascal VOC test server
expects results.
This will result in `n_classes` text files, where each file contains the predictions for one class.
Arguments:
model (Keras model): A Keras SSD model object.
img_height (int): The input image height for the model.
img_width (int): The input image width for the model.
data_generator (DataGenerator): A `DataGenerator` object with the evaluation dataset.
batch_size (int): The batch size for the evaluation.
data_generator_mode (str, optional): Either of 'resize' or 'pad'. If 'resize', the input images will
be resized (i.e. warped) to `(img_height, img_width)`. This mode does not preserve the aspect ratios of the images.
If 'pad', the input images will be first padded so that they have the aspect ratio defined by `img_height`
and `img_width` and then resized to `(img_height, img_width)`. This mode preserves the aspect ratios of the images.
classes (list or dict, optional): A list or dictionary maps the consecutive class IDs predicted by the model
their respective name strings. The list must contain the background class for class ID zero.
out_file_prefix (str, optional): A prefix for the output text file names. The suffix to each output text file name will
be the respective class name followed by the `.txt` file extension. This string is also how you specify the directory
in which the results are to be saved.
model_mode (str, optional): The mode in which the model was created, i.e. 'training', 'inference' or 'inference_fast'.
This is needed in order to know whether the model output is already decoded or still needs to be decoded. Refer to
the model documentation for the meaning of the individual modes.
confidence_thresh (float, optional): A float in [0,1), the minimum classification confidence in a specific
positive class in order to be considered for the non-maximum suppression stage for the respective class.
A lower value will result in a larger part of the selection process being done by the non-maximum suppression
stage, while a larger value will result in a larger part of the selection process happening in the confidence
thresholding stage.
iou_threshold (float, optional): A float in [0,1]. All boxes with a Jaccard similarity of greater than `iou_threshold`
with a locally maximal box will be removed from the set of predictions for a given class, where 'maximal' refers
to the box score.
top_k (int, optional): The number of highest scoring predictions to be kept for each batch item after the
non-maximum suppression stage. Defaults to 200, following the paper.
input_coords (str, optional): The box coordinate format that the model outputs. Can be either 'centroids'
for the format `(cx, cy, w, h)` (box center coordinates, width, and height), 'minmax' for the format
`(xmin, xmax, ymin, ymax)`, or 'corners' for the format `(xmin, ymin, xmax, ymax)`.
normalize_coords (bool, optional): Set to `True` if the model outputs relative coordinates (i.e. coordinates in [0,1])
and you wish to transform these relative coordinates back to absolute coordinates. If the model outputs
relative coordinates, but you do not want to convert them back to absolute coordinates, set this to `False`.
Do not set this to `True` if the model already outputs absolute coordinates, as that would result in incorrect
coordinates. Requires `img_height` and `img_width` if set to `True`.
Returns:
None.
'''
convert_to_3_channels = ConvertTo3Channels()
resize = Resize(height=img_height, width=img_width)
if data_generator_mode == 'resize':
transformations = [convert_to_3_channels, resize]
elif data_generator_mode == 'pad':
random_pad = RandomPadFixedAR(patch_aspect_ratio=img_width /
img_height,
clip_boxes=False)
transformations = [convert_to_3_channels, random_pad, resize]
else:
raise ValueError(
"Unexpected argument value: `data_generator_mode` can be either of 'resize' or 'pad', but received '{}'."
.format(data_generator_mode))
# Set the generator parameters.
generator = data_generator.generate(
batch_size=batch_size,
shuffle=False,
transformations=transformations,
label_encoder=None,
returns={'processed_images', 'image_ids', 'inverse_transform'},
keep_images_without_gt=True)
# We have to generate a separate results file for each class.
results = []
for i in range(1, len(classes)):
# Create one text file per class and put it in our results list.
results.append(
open('{}{}.txt'.format(out_file_prefix, classes[i]), 'w'))
# Compute the number of batches to iterate over the entire dataset.
n_images = data_generator.get_dataset_size()
print("Number of images in the evaluation dataset: {}".format(n_images))
n_batches = int(ceil(n_images / batch_size))
# Loop over all batches.
tr = trange(n_batches, file=sys.stdout)
tr.set_description('Producing results files')
for j in tr:
# Generate batch.
batch_X, batch_image_ids, batch_inverse_transforms = next(generator)
# Predict.
y_pred = model.predict(batch_X)
# If the model was created in 'training' mode, the raw predictions need to
# be decoded and filtered, otherwise that's already taken care of.
if model_mode == 'training':
# Decode.
y_pred = decode_y(y_pred,
confidence_thresh=confidence_thresh,
iou_threshold=iou_threshold,
top_k=top_k,
input_coords=pred_coords,
normalize_coords=normalize_coords,
img_height=img_height,
img_width=img_width)
else:
# Filter out the all-zeros dummy elements of `y_pred`.
y_pred_filtered = []
for i in range(len(y_pred)):
y_pred_filtered.append(y_pred[i][y_pred[i, :, 0] != 0])
y_pred = y_pred_filtered
# Convert the predicted box coordinates for the original images.
y_pred = apply_inverse_transforms(y_pred, batch_inverse_transforms)
# Convert each predicted box into the results format.
for k, batch_item in enumerate(y_pred):
for box in batch_item:
image_id = batch_image_ids[k]
class_id = int(box[0])
# Round the box coordinates to reduce the file size.
confidence = str(round(box[1], 4))
xmin = str(round(box[2], 1))
ymin = str(round(box[3], 1))
xmax = str(round(box[4], 1))
ymax = str(round(box[5], 1))
prediction = [image_id, confidence, xmin, ymin, xmax, ymax]
prediction_txt = ' '.join(prediction) + '\n'
# Write the predicted box to the text file for its class.
results[class_id - 1].write(prediction_txt)
# Close all the files.
for results_file in results:
results_file.close()
print("All results files saved.")
