def create_model(anchors, class_names, load_pretrained=True, freeze_body=True):
detectors_mask_shape = (13, 13, 5, 1)
matching_boxes_shape = (13, 13, 5, 5)
#Create model input layers
image_input = Input(shape=(416, 416, 3))
boxes_input = Input(shape=(None, 5))
detectors_mask_input = Input(shape=detectors_mask_shape)
matching_boxes_input = Input(shape=matching_boxes_shape)
#Create model body
yolo_model = yolo_body(image_input, len(anchors), len(class_names))
topless_yolo = Model(yolo_model.input, yolo_model.layers[-2].output)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
session = tf.Session(config=config)
if load_pretrained:
# Save topless yolo:
topless_yolo_path = os.path.join('model_data', 'yolo_topless.h5')
if not os.path.exists(topless_yolo_path):
print("CREATING TOPLESS WEIGHTS FILE")
yolo_path = os.path.join('model_data', 'yolo.h5')
model_body = load_model(yolo_path)
model_body = Model(model_body.inputs, model_body.layers[-2].output)
model_body.save_weights(topless_yolo_path)
topless_yolo.load_weights(topless_yolo_path)
if freeze_body:
for layer in topless_yolo.layers:
layer.trainable = False
final_layer = Conv2D(len(anchors) * (5 + len(class_names)), (1, 1),
activation='linear')(topless_yolo.output)
model_body = Model(image_input, final_layer)
#model_body = Model(image_input, model_body.output)
with tf.device('/cpu:0'):
model_loss = Lambda(yolo_loss,
output_shape=(1, ),
name='yolo_loss',
arguments={
'anchors': anchors,
'num_classes': len(class_names)
})([
model_body.output, boxes_input,
detectors_mask_input, matching_boxes_input
])
model = Model([
model_body.input, boxes_input, detectors_mask_input,
matching_boxes_input
], model_loss)
model.summary()
return model_body, model
def create_model(anchors, class_names, load_pretrained=True, freeze_body=True):
'''
returns the body of the model and the model
# Params:
load_pretrained: whether or not to load the pretrained model or initialize all weights
freeze_body: whether or not to freeze all weights except for the last layer's
# Returns:
model_body: YOLOv2 with new output layer
model: YOLOv2 with custom loss Lambda layer
'''
detectors_mask_shape = (13, 13, 5, 1)
matching_boxes_shape = (13, 13, 5, 5)
# Create model input layers.
image_input = Input(shape=(416, 416, 3))
boxes_input = Input(shape=(None, 5))
detectors_mask_input = Input(shape=detectors_mask_shape)
matching_boxes_input = Input(shape=matching_boxes_shape)
# Create model body.
yolo_model = yolo_body(image_input, len(anchors), len(class_names))
topless_yolo = Model(yolo_model.input, yolo_model.layers[-2].output)
if load_pretrained:
# Save topless yolo:
topless_yolo_path = os.path.join('model_data', 'yolo_topless.h5')
if not os.path.exists(topless_yolo_path):
print("CREATING TOPLESS WEIGHTS FILE")
yolo_path = os.path.join('model_data', 'yolo.h5')
model_body = load_model(yolo_path)
model_body = Model(model_body.inputs, model_body.layers[-2].output)
model_body.save_weights(topless_yolo_path)
topless_yolo.load_weights(topless_yolo_path)
if freeze_body:
for layer in topless_yolo.layers:
layer.trainable = False
final_layer = Conv2D(len(anchors)*(5+len(class_names)), (1, 1), activation='linear')(topless_yolo.output)
model_body = Model(image_input, final_layer)
# Place model loss on CPU to reduce GPU memory usage.
with tf.device('/cpu:0'):
# TODO: Replace Lambda with custom Keras layer for loss.
model_loss = Lambda(
yolo_loss,
output_shape=(1, ),
name='yolo_loss',
arguments={'anchors': anchors,
'num_classes': len(class_names)})([
model_body.output, boxes_input,
detectors_mask_input, matching_boxes_input
])
model = Model(
[model_body.input, boxes_input, detectors_mask_input,
matching_boxes_input], model_loss)
return model_body, model
def create_model(anchors, class_names, load_pretrained=True, freeze_body=True):
'''
returns the body of the model and the model
# Params:
load_pretrained: whether or not to load the pretrained model or initialize all weights
freeze_body: whether or not to freeze all weights except for the last layer's
# Returns:
model_body: YOLOv2 with new output layer
model: YOLOv2 with custom loss Lambda layer
'''
detectors_mask_shape = (13, 13, 5, 1)
matching_boxes_shape = (13, 13, 5, 4)
# Create model input layers.
image_input = Input(shape=(416, 416, 3))
# Create model body.
yolo_model = yolo_body(image_input, len(anchors), len(class_names))
topless_yolo = Model(yolo_model.input, yolo_model.layers[-2].output)
final_layer = Conv2D(len(anchors) * (5), (1, 1),
activation='linear')(topless_yolo.output)
model_body = Model(image_input, final_layer)
return model_body
def create_model(anchors, class_names, load_pretrained=True, freeze_body=True):
"""
Create and return the body of the model and the model
:param anchors: list of anchors from the annotation
:param class_names: list of class names from classes.txt file
:param load_pretrained: bool to confirm whether or not to load the pretrained model and initialize all weights.
:param freeze_body: bool to confirm whether or not to freeze all weights except for the last layer
:return: - model_body: YOLO with new output layer
- model: YOLO with custom loss Lambda layer
"""
detectors_mask_shape = (13, 13, 5, 1)
matching_boxes_shape = (13, 13, 5, 5)
# Create model input layers
image_input = Input(shape=(416, 416, 3))
boxes_input = Input(shape=(None, 5))
detectors_mask_input = Input(shape=detectors_mask_shape)
matching_boxes_input = Input(shape=matching_boxes_shape)
# Create model body
yolo_model = yolo_body(image_input, len(anchors), len(class_names))
topless_yolo = Model(yolo_model.input, yolo_model.layers[-2].output)
# Check if pretrained
if load_pretrained:
# Save topless YOLO
topless_yolo_path = os.path.join('model_data', 'yolo_topless.h5')
if not os.path.exists(topless_yolo_path):
print("[INFO] Creating Topless YOLO Weights File...")
yolo_path = os.path.join('model_data', 'yolo.h5')
model_body = load_model(yolo_path)
model_body = Model(model_body.inputs, model_body.layers[-2].output)
model_body.save_weights(topless_yolo_path)
topless_yolo.load_weights(topless_yolo_path)
# Check if freeze all weights
if freeze_body:
for layer in topless_yolo.layers:
layer.trainable = False
final_layer = Conv2D(len(anchors) * (5 + len(class_names)), (1, 1),
activation='linear')(topless_yolo.output)
model_body = Model(image_input, final_layer)
# Place model loss on CPU to reduce GPU memory usage
with tf.device('/cpu:0'):
model_loss = Lambda(yolo_loss,
output_shape=(1, ),
name='yolo_loss',
arguments={
'anchors': anchors,
'num_classes': len(class_names)
})([
model_body.output, boxes_input,
detectors_mask_input, matching_boxes_input
])
# Create new model
model = Model([
model_body.input, boxes_input, detectors_mask_input,
matching_boxes_input
], model_loss)
return model_body, model
def create_model(anchors, class_num):
# Create model input layers.
image_input = Input(shape=(416, 416, 3))
yolo_model = yolo_body(image_input, len(anchors), class_num)
topless_yolo = Model(yolo_model.input, yolo_model.layers[-2].output)
final_layer = Conv2D(len(anchors) * (5 + class_num), (1, 1),
activation='linear')(topless_yolo.output)
model_body = Model(image_input, final_layer)
return model_body
def create_model_body(anchors, class_names, load_pretrained=True, freeze_body=True):
'''
returns the body of the model and the model
# Params:
load_pretrained: whether or not to load the pretrained model or initialize all weights
freeze_body: whether or not to freeze all weights except for the last layer's
# Returns:
model_body: YOLOv2 with new output layer
model: YOLOv2 with custom loss Lambda layer
'''
detectors_mask_shape = (13, 13, 5, 1)
matching_boxes_shape = (13, 13, 5, 5)
# Create model input layers.
image_input = Input(shape=(416, 416, 3))
boxes_input = Input(shape=(None, 5))
detectors_mask_input = Input(shape=detectors_mask_shape)
matching_boxes_input = Input(shape=matching_boxes_shape)
# Create model body.
yolo_model = yolo_body(image_input, len(anchors), len(class_names))
topless_yolo = Model(yolo_model.input, yolo_model.layers[-2].output)
if load_pretrained:
# Save topless yolo:
topless_yolo_path = os.path.join('model_data', 'yolo_topless.h5')
if not os.path.exists(topless_yolo_path):
print("CREATING TOPLESS WEIGHTS FILE")
yolo_path = os.path.join('model_data', 'yolo.h5')
model_body = load_model(yolo_path)
model_body = Model(model_body.inputs, model_body.layers[-2].output)
model_body.save_weights(topless_yolo_path)
topless_yolo.load_weights(topless_yolo_path)
if freeze_body:
for layer in topless_yolo.layers:
layer.trainable = False
final_layer = Conv2D(len(anchors)*(5+len(class_names)), (1, 1), activation='linear')(topless_yolo.output)
model_body = Model(image_input, final_layer)
return model_body
def create_model(anchors, class_names, load_pretrained='model_data/yolo.h5'):
detectors_mask_shape = (13, 13, 5, 1)
matching_boxes_shape = (13, 13, 5, 5)
# Create model input layers.
image_input = Input(shape=(608, 608, 3))
# Create model body.
yolo_model = yolo_body(image_input, len(anchors), len(class_names))
# topless_yolo = Model(yolo_model.input, yolo_model.layers[-2].output)
model_body = yolo_model
model_body.load_weights(load_pretrained)
return model_body
def _main():
voc_path = os.path.expanduser(
'/Users/ss/Data/VOCdevkit/pascal_voc_07_07.hdf5')
classes_path = os.path.expanduser('model_data/pascal_classes.txt')
with open(classes_path) as f:
class_names = f.readlines()
class_names = [c.strip() for c in class_names]
voc = h5py.File(voc_path, 'r')
image = PIL.Image.open(io.BytesIO(voc['train/images'][28]))
orig_size = np.array([image.width, image.height])
orig_size = np.expand_dims(orig_size, axis=0)
# Image preprocessing.
image = image.resize((416, 416), PIL.Image.BICUBIC)
image_data = np.array(image, dtype=np.float)
image_data /= 255.
# Box preprocessing.
# Original boxes stored as 1D list of class, x_min, y_min, x_max, y_max.
boxes = voc['train/boxes'][28]
boxes = boxes.reshape((-1, 5))
# Get extents as y_min, x_min, y_max, x_max, class for comparision with
# model output.
boxes_extents = boxes[:, [2, 1, 4, 3, 0]]
# Get box parameters as x_center, y_center, box_width, box_height, class.
boxes_xy = 0.5 * (boxes[:, 3:5] + boxes[:, 1:3])
boxes_wh = boxes[:, 3:5] - boxes[:, 1:3]
boxes_xy = boxes_xy / orig_size
boxes_wh = boxes_wh / orig_size
boxes = np.concatenate((boxes_xy, boxes_wh, boxes[:, 0:1]), axis=1)
# Precompute detectors_mask and matching_true_boxes for training.
# Detectors mask is 1 for each spatial position in the final conv layer and
# anchor that should be active for the given boxes and 0 otherwise.
# Matching true boxes gives the regression targets for the ground truth box
# that caused a detector to be active or 0 otherwise.
anchors = COCO_ANCHORS
detectors_mask_shape = (13, 13, 5, 1)
matching_boxes_shape = (13, 13, 5, 5)
detectors_mask, matching_true_boxes = preprocess_true_boxes(
boxes, anchors, [416, 416])
# Create model input layers.
image_input = Input(shape=(416, 416, 3))
boxes_input = Input(shape=(None, 5))
detectors_mask_input = Input(shape=detectors_mask_shape)
matching_boxes_input = Input(shape=matching_boxes_shape)
print(boxes)
print(boxes_extents)
print(np.where(detectors_mask == 1)[:-1])
print(matching_true_boxes[np.where(detectors_mask == 1)[:-1]])
# Create model body.
model_body = yolo_body(image_input, len(anchors), len(class_names))
model_body = Model(image_input, model_body.output)
# Place model loss on CPU to reduce GPU memory usage.
with tf.device('/cpu:0'):
# TODO: Replace Lambda with custom Keras layer for loss.
model_loss = Lambda(yolo_loss,
output_shape=(1, ),
name='yolo_loss',
arguments={
'anchors': anchors,
'num_classes': len(class_names)
})([
model_body.output, boxes_input,
detectors_mask_input, matching_boxes_input
])
model = Model(
[image_input, boxes_input, detectors_mask_input, matching_boxes_input],
model_loss)
model.compile(
optimizer='adam', loss={
'yolo_loss': lambda y_true, y_pred: y_pred
}) # This is a hack to use the custom loss function in the last layer.
# Add batch dimension for training.
image_data = np.expand_dims(image_data, axis=0)
boxes = np.expand_dims(boxes, axis=0)
detectors_mask = np.expand_dims(detectors_mask, axis=0)
matching_true_boxes = np.expand_dims(matching_true_boxes, axis=0)
num_steps = 1000
# TODO: For full training, put preprocessing inside training loop.
# for i in range(num_steps):
# loss = model.train_on_batch(
# [image_data, boxes, detectors_mask, matching_true_boxes],
# np.zeros(len(image_data)))
model.fit([image_data, boxes, detectors_mask, matching_true_boxes],
np.zeros(len(image_data)),
batch_size=1,
epochs=num_steps)
model.save_weights('overfit_weights.h5')
# Create output variables for prediction.
yolo_outputs = yolo_head(model_body.output, anchors, len(class_names))
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(yolo_outputs,
input_image_shape,
score_threshold=.3,
iou_threshold=.9)
# Run prediction on overfit image.
sess = K.get_session() # TODO: Remove dependence on Tensorflow session.
out_boxes, out_scores, out_classes = sess.run(
[boxes, scores, classes],
feed_dict={
model_body.input: image_data,
input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
print('Found {} boxes for image.'.format(len(out_boxes)))
print(out_boxes)
# Plot image with predicted boxes.
image_with_boxes = draw_boxes(image_data[0], out_boxes, out_classes,
class_names, out_scores)
plt.imshow(image_with_boxes, interpolation='nearest')
plt.show()
def create_model(anchors, class_names, load_pretrained=True, num_frozen=0):
'''
returns the body of the model and the model
# Params:
load_pretrained: whether or not to load the pretrained model or initialize all weights
num_frozen: number of layers whose weights are held frozen during training
# Returns:
model_body: YOLOv2 with new output layer
model: YOLOv2 with custom loss Lambda layer
'''
detectors_mask_shape = (19, 19, 5, 1)
matching_boxes_shape = (19, 19, 5, 5)
# Create model input layers.
image_input = Input(shape=(608, 608, 3))
boxes_input = Input(shape=(None, 5))
detectors_mask_input = Input(shape=detectors_mask_shape)
matching_boxes_input = Input(shape=matching_boxes_shape)
# Create model body.
yolo_model = yolo_body(image_input, len(anchors), len(class_names))
topless_yolo = Model(yolo_model.input, yolo_model.layers[-2].output)
if load_pretrained:
# Save topless yolo:
topless_yolo_path = os.path.join('data', 'model_data',
'yolo_topless.h5')
if not os.path.exists(topless_yolo_path):
print("CREATING TOPLESS WEIGHTS FILE")
yolo_path = os.path.join('data', 'model_data', 'yolo2.h5')
model_body = load_model(yolo_path)
model_body = Model(model_body.inputs, model_body.layers[-2].output)
model_body.save_weights(topless_yolo_path)
topless_yolo.load_weights(topless_yolo_path)
num_layers = len(topless_yolo.layers)
if num_frozen > num_layers:
print "Cannot freeze", num_frozen, "/", num_layers, "layers"
sys.exit()
print "Freezing", num_frozen, "/", num_layers, "layers"
for i, layer in enumerate(topless_yolo.layers):
if i < num_frozen:
layer.trainable = False
final_layer = Conv2D(len(anchors) * (5 + len(class_names)), (1, 1),
activation='linear')(topless_yolo.output)
model_body = Model(image_input, final_layer)
# Place model loss on CPU to reduce GPU memory usage.
with tf.device('/cpu:0'):
# TODO: Replace Lambda with custom Keras layer for loss.
model_loss = Lambda(yolo_loss,
output_shape=(1, ),
name='yolo_loss',
arguments={
'anchors': anchors,
'num_classes': len(class_names)
})([
model_body.output, boxes_input,
detectors_mask_input, matching_boxes_input
])
model = Model([
model_body.input, boxes_input, detectors_mask_input,
matching_boxes_input
], model_loss)
return model_body, model
def create_model(anchors, class_names, load_pretrained=True, freeze_body=True):
'''
returns the body of the model and the model
# Params:
load_pretrained: whether or not to load the pretrained model or initialize all weights
freeze_body: whether or not to freeze all weights except for the last layer's
# Returns:
model_body: YOLOv2 with new output layer
model: YOLOv2 with custom loss Lambda layer
'''
detectors_mask_shape = (13, 13, 5, 1)
matching_boxes_shape = (13, 13, 5, 5)
# Create model input layers.
image_input = Input(shape=(416, 416, 3))
boxes_input = Input(shape=(None, 5))
detectors_mask_input = Input(shape=detectors_mask_shape)
matching_boxes_input = Input(shape=matching_boxes_shape)
# Create model body.
yolo_model = yolo_body(image_input, len(anchors), len(class_names))
topless_yolo = Model(yolo_model.input, yolo_model.layers[-2].output)
if load_pretrained:
# Save topless yolo:
topless_yolo_path = os.path.join('model_data', 'yolo_topless.h5')
if not os.path.exists(topless_yolo_path):
print("CREATING TOPLESS WEIGHTS FILE")
yolo_path = os.path.join('model_data', 'yolo.h5')
model_body = load_model(yolo_path)
model_body = Model(model_body.inputs, model_body.layers[-2].output)
model_body.save_weights(topless_yolo_path)
topless_yolo.load_weights(topless_yolo_path)
if freeze_body:
for layer in topless_yolo.layers:
layer.trainable = False
final_layer = Conv2D(len(anchors)*(5+len(class_names)), (1, 1), activation='linear')(topless_yolo.output)
model_body = Model(image_input, final_layer)
# Place model loss on CPU to reduce GPU memory usage.
with tf.device('/cpu:0'):
# TODO: Replace Lambda with custom Keras layer for loss.
model_loss = Lambda(
yolo_loss,
output_shape=(1, ),
name='yolo_loss',
arguments={'anchors': anchors,
'num_classes': len(class_names)})([
model_body.output, boxes_input,
detectors_mask_input, matching_boxes_input
])
model = Model(
[model_body.input, boxes_input, detectors_mask_input,
matching_boxes_input], model_loss)
return model_body, model
def _main(args):
#訓練データセットに対する情報の読み込みfrom voc_to_hdf5
voc_path = os.path.expanduser(args.data_path)
#ラベル情報.txtへのpath
classes_path = os.path.expanduser(args.classes_path)
#アスペクト比の格納？
anchors_path = os.path.expanduser(args.anchors_path)
#クラス名のリストを作成
with open(classes_path) as f:
class_names = f.readlines()
class_names = [c.strip() for c in class_names]
if os.path.isfile(anchors_path):
with open(anchors_path) as f:
anchors = f.readline()
anchors = [float(x) for x in anchors.split(',')]
anchors = np.array(anchors).reshape(-1, 2)
else:
anchors = YOLO_ANCHORS
voc = h5py.File(voc_path, 'r')
image = PIL.Image.open(io.BytesIO(voc['test/images'][0]))
orig_size = np.array([image.width, image.height])
orig_size = np.expand_dims(orig_size, axis=0)
# Image preprocessing.
image = image.resize((416, 416), PIL.Image.BICUBIC)
image_data = np.array(image, dtype=np.float)
image_data /= 255.
# Box preprocessing.
# Original boxes stored as 1D list of class, x_min, y_min, x_max, y_max.
boxes = voc['test/boxes'][0]
boxes = boxes.reshape((-1, 5))
# Get extents as y_min, x_min, y_max, x_max, class for comparision with
# model output.
boxes_extents = boxes[:, [2, 1, 4, 3, 0]]
# Get box parameters as x_center, y_center, box_width, box_height, class.
boxes_xy = 0.5 * (boxes[:, 3:5] + boxes[:, 1:3])
boxes_wh = boxes[:, 3:5] - boxes[:, 1:3]
boxes_xy = boxes_xy / orig_size
boxes_wh = boxes_wh / orig_size
#annotation部分のxy座標、幅
boxes = np.concatenate((boxes_xy, boxes_wh, boxes[:, 0:1]), axis=1)
# Precompute detectors_mask and matching_true_boxes for training.
# Detectors mask is 1 for each spatial position in the final conv layer and
# anchor that should be active for the given boxes and 0 otherwise.
# Matching true boxes gives the regression targets for the ground truth box
# that caused a detector to be active or 0 otherwise.
detectors_mask_shape = (13, 13, 5, 1)
matching_boxes_shape = (13, 13, 5, 5)
detectors_mask, matching_true_boxes = preprocess_true_boxes(
boxes, anchors, [416, 416])
# Create model input layers.
image_input = Input(shape=(416, 416, 3))
boxes_input = Input(shape=(None, 5))
detectors_mask_input = Input(shape=detectors_mask_shape)
matching_boxes_input = Input(shape=matching_boxes_shape)
print('Boxes:')
print(boxes)
print('Box corners:')
print(boxes_extents)
print('Active detectors:')
print(np.where(detectors_mask == 1)[:-1])
print('Matching boxes for active detectors:')
print(matching_true_boxes[np.where(detectors_mask == 1)[:-1]])
# Create model body.
model_body = yolo_body(image_input, len(anchors), len(class_names))
model_body = Model(image_input, model_body.output)
# Place model loss on CPU to reduce GPU memory usage.
with tf.device('/cpu:0'):
# TODO: Replace Lambda with custom Keras layer for loss.
model_loss = Lambda(yolo_loss,
output_shape=(1, ),
name='yolo_loss',
arguments={
'anchors': anchors,
'num_classes': len(class_names)
})([
model_body.output, boxes_input,
detectors_mask_input, matching_boxes_input
])
model = Model(
[image_input, boxes_input, detectors_mask_input, matching_boxes_input],
model_loss)
model.compile(
optimizer='adam', loss={
'yolo_loss': lambda y_true, y_pred: y_pred
}) # This is a hack to use the custom loss function in the last layer.
# Add batch dimension for training.
image_data = np.expand_dims(image_data, axis=0)
boxes = np.expand_dims(boxes, axis=0)
detectors_mask = np.expand_dims(detectors_mask, axis=0)
matching_true_boxes = np.expand_dims(matching_true_boxes, axis=0)
num_steps = 50
#checkpoint = ModelCheckpoint('model1.h5', monitor='val_loss', verbose=1,
#save_best_only=True, save_weights_only=False, mode='auto', period=1)
# TODO: For full training, put preprocessing inside training loop.
# for i in range(num_steps):
# loss = model.train_on_batch(
# [image_data, boxes, detectors_mask, matching_true_boxes],
# np.zeros(len(image_data)))
model.fit(
[image_data, boxes, detectors_mask, matching_true_boxes],
np.zeros(len(image_data)),
batch_size=1,
epochs=num_steps,
#callbacks=[checkpoint]
)
model.save('model.h5')
model.save_weights('overfit_weights.h5')
#model_json_str = model.to_json()
#open('mlp_model.json', 'w').write(model_json_str)
#model.save_weights('mlp_weights.h5');
# Create output variables for prediction.
yolo_outputs = yolo_head(model_body.output, anchors, len(class_names))
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(yolo_outputs,
input_image_shape,
score_threshold=.3,
iou_threshold=.9)
# Run prediction on overfit image.
sess = K.get_session() # TODO: Remove dependence on Tensorflow session.
out_boxes, out_scores, out_classes = sess.run(
[boxes, scores, classes],
feed_dict={
model_body.input: image_data,
input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
print('Found {} boxes for image.'.format(len(out_boxes)))
print(out_boxes)
# Plot image with predicted boxes.
image_with_boxes = draw_boxes(image_data[0], out_boxes, out_classes,
class_names, out_scores)
plt.imshow(image_with_boxes, interpolation='nearest')
plt.show()
def _main(args):
voc_path = os.path.expanduser(args.data_path)
#classes_path = os.path.expanduser(args.classes_path)
anchors_path = os.path.expanduser(args.anchors_path)
#with open(classes_path) as f:
# class_names = f.readlines()
#class_names = [c.strip() for c in class_names]
class_names = ['circle', 'square']
if os.path.isfile(anchors_path):
with open(anchors_path) as f:
anchors = f.readline()
anchors = [float(x) for x in anchors.split(',')]
anchors = np.array(anchors).reshape(-1, 2)
else:
anchors = YOLO_ANCHORS
''' file load ''
voc = h5py.File(voc_path, 'r')
image = PIL.Image.open(io.BytesIO(voc['train/images'][28]))
orig_size = np.array([image.width, image.height])
orig_size = np.expand_dims(orig_size, axis=0)
# Image preprocessing.
image = image.resize((416, 416), PIL.Image.BICUBIC)
image_data = np.array(image, dtype=np.float)
image_data /= 255.
'''
# Generate shape w/pymrt
image_data = np.empty((2, 416, 416, 3), dtype=np.float)
circ = geo.circle(shape=(416, 416), radius=40, position=0.5)
image_data[0, :, :, 0] = circ
image_data[0, :, :, 1] = circ
image_data[0, :, :, 2] = circ
sq = geo.square(shape=(416, 416), side=40, position=0.5)
image_data[1, :, :, 0] = sq
image_data[1, :, :, 1] = sq
image_data[1, :, :, 2] = sq
#image = image_data
orig_size = np.array([416, 416])
# Box preprocessing.
# Original boxes stored as 1D list of class, x_min, y_min, x_max, y_max.
## boxes = voc['train/boxes'][28]
boxes = np.asarray([[0, 168, 168, 248, 248], [1, 168, 168, 248, 248]])
boxes = boxes.reshape((-1, 5))
# Get extents as y_min, x_min, y_max, x_max, class for comparision with
# model output.
boxes_extents = boxes[:, [2, 1, 4, 3, 0]]
# Get box parameters as x_center, y_center, box_width, box_height, class.
boxes_xy = 0.5 * (boxes[:, 3:5] + boxes[:, 1:3])
boxes_wh = boxes[:, 3:5] - boxes[:, 1:3]
boxes_xy = boxes_xy / orig_size # convert to relative coordinates
boxes_wh = boxes_wh / orig_size
boxes = np.concatenate((boxes_xy, boxes_wh, boxes[:, 0:1]), axis=1)
# Precompute detectors_mask and matching_true_boxes for training.
# Detectors mask is 1 for each spatial position in the final conv layer and
# anchor that should be active for the given boxes and 0 otherwise.
# Matching true boxes gives the regression targets for the ground truth box
# that caused a detector to be active or 0 otherwise.
detectors_mask_shape = (13, 13, 5, 1)
matching_boxes_shape = (13, 13, 5, 5)
def get_detector_mask(boxes, anchors):
detectors_mask = [0 for i in range(len(boxes))]
matching_true_boxes = [0 for i in range(len(boxes))]
for i, box in enumerate(boxes):
box = box.reshape((-1, 5))
detectors_mask[i], matching_true_boxes[i] = preprocess_true_boxes(
box, anchors, [416, 416])
return np.array(detectors_mask), np.array(matching_true_boxes)
detectors_mask, matching_true_boxes = get_detector_mask(boxes, anchors)
#detectors_mask, matching_true_boxes = preprocess_true_boxes(boxes, anchors,
# [416, 416])
# Create model input layers.
image_input = Input(shape=(416, 416, 3))
boxes_input = Input(shape=(None, 5))
detectors_mask_input = Input(shape=detectors_mask_shape)
matching_boxes_input = Input(shape=matching_boxes_shape)
#import pdb; pdb.set_trace()
print('Boxes:')
print(boxes)
##print('Box corners:')
##print(boxes_extents)
print('Active detectors:')
print(np.where(detectors_mask == 1)[:-1])
print('Matching boxes for active detectors:')
print(matching_true_boxes[np.where(detectors_mask == 1)[:-1]])
# Create model body.
model_body = yolo_body(image_input, len(anchors), len(class_names))
model_body = Model(image_input, model_body.output)
# Place model loss on CPU to reduce GPU memory usage.
with tf.device('/cpu:0'):
# TODO: Replace Lambda with custom Keras layer for loss.
model_loss = Lambda(yolo_loss,
output_shape=(1, ),
name='yolo_loss',
arguments={
'anchors': anchors,
'num_classes': len(class_names)
})([
model_body.output, boxes_input,
detectors_mask_input, matching_boxes_input
])
model = Model(
[image_input, boxes_input, detectors_mask_input, matching_boxes_input],
model_loss)
model.compile(
optimizer='adam', loss={
'yolo_loss': lambda y_true, y_pred: y_pred
}) # This is a hack to use the custom loss function in the last layer.
# Add batch dimension for training.
#image_data = image_data[0,:,:,:]
#image_data = np.expand_dims(image_data, axis=0)
boxes = np.expand_dims(boxes, axis=1)
#detectors_mask = np.expand_dims(detectors_mask, axis=0)
#matching_true_boxes = np.expand_dims(matching_true_boxes, axis=0)
num_steps = 1000
# TODO: For full training, put preprocessing inside training loop.
# for i in range(num_steps):
# loss = model.train_on_batch(
# [image_data, boxes, detectors_mask, matching_true_boxes],
# np.zeros(len(image_data)))
#import pdb; pdb.set_trace()
model.fit([image_data, boxes, detectors_mask, matching_true_boxes],
np.zeros(len(image_data)),
batch_size=2,
epochs=num_steps)
model.save_weights('overfit_circle_square_weights.h5')
#model.load_weights('overfit_circle_square_weights.h5')
#model.load_weights('overfit_weights.h5')
# Create output variables for prediction.
yolo_outputs = yolo_head(model_body.output, anchors, len(class_names))
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(yolo_outputs,
input_image_shape,
score_threshold=.3,
iou_threshold=.9)
# Run prediction on overfit image.
num_shapes = 10
image_data = np.empty((1, 416, 416, 3), dtype=np.float)
#for sh in range(num_shape)
circ2 = geo.circle(shape=(416, 416), radius=40, position=(0.4, 0.4))
#circ3 = geo.circle(shape=(416,416),radius=40,position=0.8)
#sq2 = geo.square(shape=(416,416),side=40,position=0.2)
sq3 = geo.square(shape=(416, 416), side=40, position=0.8)
image_data = np.empty((1, 416, 416, 3), dtype=np.float)
image_data[0, :, :, 0] = circ2 + sq3
image_data[0, :, :, 1] = circ2 + sq3
image_data[0, :, :, 2] = circ2 + sq3
sess = K.get_session() # TODO: Remove dependence on Tensorflow session.
out_boxes, out_scores, out_classes = sess.run(
[boxes, scores, classes],
feed_dict={
model_body.input: image_data,
#input_image_shape: [image.size[1], image.size[0]],
input_image_shape: [416, 416],
K.learning_phase(): 0
})
print('Found {} boxes for image.'.format(len(out_boxes)))
print(out_boxes)
# Plot image with predicted boxes.
image_with_boxes = draw_boxes(image_data[0], out_boxes, out_classes,
class_names, out_scores)
plt.imshow(image_with_boxes, interpolation='nearest')
plt.show()
def create_model(image_size, anchors, num_classes, model_file=None, load_pretrained=True, freeze_body=True):
"""
Generates keras models of YOLOv2. Modified from original version to optionally load the whole model from file, and
work with any size image.
:param image_size: [w, h] size of the image data (square when using YOLO anchors)
:param anchors: list of tuples of anchor boxes
:param num_classes: number of classes
:param model_file: path to keras model file. If this is used, the top layer will be maintained,
but care must be taken to make sure that the model is compatible with the data
:param load_pretrained: load the pretrained model (true) or initialize all weights (false). Ignored wth model file.
:param freeze_body: freeze all weights except for the last layers. Ignored with model file.
:return model_body: keras model of YOLOv2 with modified output layer, for testing
:return model: keras model of YOLOv2 with custom loss layer, for training
Modified version of the original yad2k version in retrain_yolo.py
"""
if not model_file is None:
print("USING MODEL FILE")
model_body = load_model(model_file)
input_shape = model_body.layers[0].input_shape
image_input_size = [input_shape[2], input_shape[1]]
assert np.allclose(image_input_size, image_size)
detectors_mask_shape = (image_size[1]//32, image_size[0]//32, len(anchors), 1)
matching_boxes_shape = (image_size[1]//32, image_size[0]//32, len(anchors), 5)
# Create model input layers.
image_input = Input(shape=(image_size[1], image_size[0], 3))
boxes_input = Input(shape=(None, 5))
detectors_mask_input = Input(shape=detectors_mask_shape)
matching_boxes_input = Input(shape=matching_boxes_shape)
ids_input = Input(shape=(1,)) # added input to track IDs
if model_file is None:
# Create model body.
yolo_model = yolo_body(image_input, len(anchors), num_classes)
topless_yolo = Model(yolo_model.input, yolo_model.layers[-2].output)
if load_pretrained:
# Save topless yolo:
topless_yolo_path = os.path.join('data/model_data', 'yolo_topless.h5')
if not os.path.exists(topless_yolo_path):
print("CREATING TOPLESS WEIGHTS FILE")
yolo_path = os.path.join('data/model_data', 'yolo.h5')
model_body = load_model(yolo_path)
model_body = Model(model_body.inputs, model_body.layers[-2].output)
model_body.save_weights(topless_yolo_path)
topless_yolo.load_weights(topless_yolo_path)
final_layer = Conv2D(len(anchors)*(5+num_classes), (1, 1), activation='linear')(topless_yolo.output)
model_body = Model(image_input, final_layer)
if freeze_body:
for layer in model_body.layers[:-1]:
layer.trainable = False
# Place model loss on CPU to reduce GPU memory usage.
with tf.device('/cpu:0'):
# TODO: Replace Lambda with custom Keras layer for loss.
model_loss = Lambda(
yolo_loss,
output_shape=(1, ),
name='yolo_loss',
arguments={'anchors': anchors,
'num_classes': num_classes})([
model_body.output, boxes_input,
detectors_mask_input, matching_boxes_input
])
model = Model(
[model_body.input, boxes_input, detectors_mask_input,
matching_boxes_input, ids_input], model_loss)
return model_body, model
def create_model(anchors, class_names, load_pretrained=True, freeze_body=True):
'''
returns the body of the model and the model
# Params:
load_pretrained: whether or not to load the pretrained model or initialize all weights
freeze_body: whether or not to freeze all weights except for the last layer's
# Returns:
model_body: YOLOv2 with new output layer
model: YOLOv2 with custom loss Lambda layer
'''
detectors_mask_shape = (13, 13, 5, 1)
matching_boxes_shape = (13, 13, 5, 5)
# Create model input layers.
image_input = Input(shape=(416, 416, 3)) #
boxes_input = Input(shape=(None, 5)) #true label
detectors_mask_input = Input(
shape=detectors_mask_shape) # a return from get_detectors function
matching_boxes_input = Input(
shape=matching_boxes_shape) # a return from get_detectors function
# Create model body.
# Note:Model here will be created without last layer as we will train last layer again every time we have new dataset.
yolo_model = yolo_body(image_input, len(anchors), len(class_names))
"""
The next line creates model between input layer and the layer before the last layer but how it creates the model with only knowing the first layer and last layer of model ?
each layer has a pointer to the preeceding layer so it creates model from the last layer going upwards to the first layer through pointers
"""
topless_yolo = Model(yolo_model.input, yolo_model.layers[-2].output)
if load_pretrained:
# Save topless yolo:
# Saving occurs only once to save model without last layer.
#every time after this time we saved in, we will only load data.
topless_yolo_path = os.path.join(
'model_data', 'yolo_topless.h5') #get path of topless_yolo
if not os.path.exists(
topless_yolo_path
): # ask if topless_yolo exists ? if exists then only load it ,if not exists then enter the if conditional to load the full yolo model
print("CREATING TOPLESS WEIGHTS FILE"
) #so that you can extract the topless_yolo model.
yolo_path = os.path.join('model_data', 'yolo.h5')
model_body = load_model(yolo_path)
model_body = Model(model_body.inputs, model_body.layers[-2].output)
model_body.save_weights(topless_yolo_path)
topless_yolo.load_weights(topless_yolo_path)
#freezing our model as we don't need it now as we have already pretrained weights representing this model
if freeze_body:
for layer in topless_yolo.layers:
layer.trainable = False
final_layer = Conv2D(len(anchors) * (5 + len(class_names)), (1, 1),
activation='linear')(topless_yolo.output)
model_body = Model(image_input, final_layer)
# Place model loss on CPU to reduce GPU memory usage.
"""
model_loss is a layer to represent the loss of our model
so it feeds the final layer outputs to yolo_loss function to calculate loss and then passes it to Model func. to generate
new model from the input to the last layer which is now the loss layer.
"""
with tf.device('/cpu:0'):
# TODO: Replace Lambda with custom Keras layer for loss.
model_loss = Lambda(yolo_loss,
output_shape=(1, ),
name='yolo_loss',
arguments={
'anchors': anchors,
'num_classes': len(class_names)
})([
model_body.output, boxes_input,
detectors_mask_input, matching_boxes_input
])
model = Model([
model_body.input, boxes_input, detectors_mask_input,
matching_boxes_input
], model_loss)
return model_body, model
def train_yolo_model(numb_image=10,
learning_rate=0.00005,
num_epochs=100,
minibatch_size=5,
print_cost=True,
model_path="./model_data/yolo_trained_model.h5"):
""" Create and train yolo model
Parameters:
-----------
numb_image : int
number of images used for training
learning_rate : float
learning rate of the optimization
num_epochs : int
number of epochs of the optimization loop
minibatch_size : int
size of a minibatch
print_cost : boolean
True to print the cost every 5 epochs
model_path : string
location to save trained model
Returns:
--------
costs : list
list of cost values
"""
costs = [] # To keep track of the cost
# Load data
image_size = (608, 608)
dataset = load_data(numb_image, "model_data/true_boxes.h5")
# Create model
X = Input(batch_shape=[None, 608, 608, 3], dtype=tf.float32)
Y1 = tf.placeholder(shape=[None, 10, 5], dtype=tf.float32) # true boxes
Y2 = tf.placeholder(shape=[None, 19, 19, 5, 1],
dtype=tf.float32) # detector mask
Y3 = tf.placeholder(shape=[None, 19, 19, 5, 5],
dtype=tf.float32) # matching true boxes
yolo_m = yolo_body(inputs=X,
num_anchors=len(anchors),
num_classes=len(class_names))
args = [yolo_m.output, Y1, Y2, Y3]
cost = yolo_loss(args, anchors, len(class_names))
# Backpropagation: Define the tensorflow optimizer. Use an AdamOptimizer that minimizes the cost.
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)
# Initialize all the variables globally
init = tf.global_variables_initializer()
# Run session to train network
with tf.Session() as sess:
# Run initialization
sess.run(init)
# # Run training loop
for epoch in range(num_epochs):
minibatch_cost = 0.
num_minibatches = int(
numb_image / minibatch_size
) # number of minibatches of size minibatch_size in the train set
minibatches = random_mini_batches(dataset['X'], dataset['Y'],
minibatch_size)
for minibatch in minibatches:
# Select a minibatch
(minibatch_X, minibatch_Y) = minibatch
# Preprocess true boxes
detectors_mask = []
matching_true_boxes = []
for i in range(0, minibatch_Y.shape[0]):
detectors_mask_tmp, matching_true_boxes_tmp = preprocess_true_boxes(
minibatch_Y[i, :, :], anchors, image_size)
if len(detectors_mask) == 0:
detectors_mask = np.expand_dims(detectors_mask_tmp,
axis=0)
else:
detectors_mask = np.append(detectors_mask,
np.expand_dims(
detectors_mask_tmp,
axis=0),
axis=0)
if len(matching_true_boxes) == 0:
matching_true_boxes = np.expand_dims(
matching_true_boxes_tmp, axis=0)
else:
matching_true_boxes = np.append(
matching_true_boxes,
np.expand_dims(matching_true_boxes_tmp, axis=0),
axis=0)
# Run the session to execute the optimizer and the cost
_, temp_cost = sess.run(
[optimizer, cost],
feed_dict={
X: minibatch_X,
Y1: minibatch_Y,
Y2: detectors_mask,
Y3: matching_true_boxes
})
minibatch_cost += temp_cost / num_minibatches
# Print the cost every epoch
if print_cost == True and epoch % 5 == 0:
print("Cost after epoch %i: %f" % (epoch, minibatch_cost))
if print_cost == True and epoch % 1 == 0:
costs.append(minibatch_cost)
yolo_m.save_weights(model_path)
return costs
def _main(args):
voc_path = os.path.expanduser(args.data_path)
classes_path = os.path.expanduser(args.classes_path)
anchors_path = os.path.expanduser(args.anchors_path)
with open(classes_path) as f:
class_names = f.readlines()
class_names = [c.strip() for c in class_names]
if os.path.isfile(anchors_path):
with open(anchors_path) as f:
anchors = f.readline()
anchors = [float(x) for x in anchors.split(',')]
anchors = np.array(anchors).reshape(-1, 2)
else:
anchors = YOLO_ANCHORS
voc = h5py.File(voc_path, 'r')
image = PIL.Image.open(io.BytesIO(voc['train/images'][28]))
orig_size = np.array([image.width, image.height])
orig_size = np.expand_dims(orig_size, axis=0)
# Image preprocessing.
image = image.resize((416, 416), PIL.Image.BICUBIC)
image_data = np.array(image, dtype=np.float)
image_data /= 255.
# Box preprocessing.
# Original boxes stored as 1D list of class, x_min, y_min, x_max, y_max.
boxes = voc['train/boxes'][28]
boxes = boxes.reshape((-1, 5))
# Get extents as y_min, x_min, y_max, x_max, class for comparision with
# model output.
boxes_extents = boxes[:, [2, 1, 4, 3, 0]]
# Get box parameters as x_center, y_center, box_width, box_height, class.
boxes_xy = 0.5 * (boxes[:, 3:5] + boxes[:, 1:3])
boxes_wh = boxes[:, 3:5] - boxes[:, 1:3]
boxes_xy = boxes_xy / orig_size
boxes_wh = boxes_wh / orig_size
boxes = np.concatenate((boxes_xy, boxes_wh, boxes[:, 0:1]), axis=1)
# Precompute detectors_mask and matching_true_boxes for training.
# Detectors mask is 1 for each spatial position in the final conv layer and
# anchor that should be active for the given boxes and 0 otherwise.
# Matching true boxes gives the regression targets for the ground truth box
# that caused a detector to be active or 0 otherwise.
detectors_mask_shape = (13, 13, 5, 1)
matching_boxes_shape = (13, 13, 5, 5)
detectors_mask, matching_true_boxes = preprocess_true_boxes(boxes, anchors,
[416, 416])
# Create model input layers.
image_input = Input(shape=(416, 416, 3))
boxes_input = Input(shape=(None, 5))
detectors_mask_input = Input(shape=detectors_mask_shape)
matching_boxes_input = Input(shape=matching_boxes_shape)
print('Boxes:')
print(boxes)
print('Box corners:')
print(boxes_extents)
print('Active detectors:')
print(np.where(detectors_mask == 1)[:-1])
print('Matching boxes for active detectors:')
print(matching_true_boxes[np.where(detectors_mask == 1)[:-1]])
# Create model body.
model_body = yolo_body(image_input, len(anchors), len(class_names))
model_body = Model(image_input, model_body.output)
# Place model loss on CPU to reduce GPU memory usage.
with tf.device('/cpu:0'):
# TODO: Replace Lambda with custom Keras layer for loss.
model_loss = Lambda(
yolo_loss,
output_shape=(1, ),
name='yolo_loss',
arguments={'anchors': anchors,
'num_classes': len(class_names)})([
model_body.output, boxes_input,
detectors_mask_input, matching_boxes_input
])
model = Model(
[image_input, boxes_input, detectors_mask_input,
matching_boxes_input], model_loss)
model.compile(
optimizer='adam', loss={
'yolo_loss': lambda y_true, y_pred: y_pred
}) # This is a hack to use the custom loss function in the last layer.
# Add batch dimension for training.
image_data = np.expand_dims(image_data, axis=0)
boxes = np.expand_dims(boxes, axis=0)
detectors_mask = np.expand_dims(detectors_mask, axis=0)
matching_true_boxes = np.expand_dims(matching_true_boxes, axis=0)
num_steps = 1000
# TODO: For full training, put preprocessing inside training loop.
# for i in range(num_steps):
# loss = model.train_on_batch(
# [image_data, boxes, detectors_mask, matching_true_boxes],
# np.zeros(len(image_data)))
model.fit([image_data, boxes, detectors_mask, matching_true_boxes],
np.zeros(len(image_data)),
batch_size=1,
epochs=num_steps)
model.save_weights('overfit_weights.h5')
# Create output variables for prediction.
yolo_outputs = yolo_head(model_body.output, anchors, len(class_names))
input_image_shape = K.placeholder(shape=(2, ))
boxes, scores, classes = yolo_eval(
yolo_outputs, input_image_shape, score_threshold=.3, iou_threshold=.9)
# Run prediction on overfit image.
sess = K.get_session() # TODO: Remove dependence on Tensorflow session.
out_boxes, out_scores, out_classes = sess.run(
[boxes, scores, classes],
feed_dict={
model_body.input: image_data,
input_image_shape: [image.size[1], image.size[0]],
K.learning_phase(): 0
})
print('Found {} boxes for image.'.format(len(out_boxes)))
print(out_boxes)
# Plot image with predicted boxes.
image_with_boxes = draw_boxes(image_data[0], out_boxes, out_classes,
class_names, out_scores)
plt.imshow(image_with_boxes, interpolation='nearest')
plt.show()