def create_proposal_target_layer(rpn_rois, scaled_gt_boxes, num_classes):
'''
Creates a proposal target layer that is used for training an object detection network as proposed in the "Faster R-CNN" paper:
Shaoqing Ren and Kaiming He and Ross Girshick and Jian Sun:
"Faster R-CNN: Towards Real-Time Object Detection with Region Proposal Networks"
Assigns object detection proposals to ground-truth targets.
Produces proposal classification labels and bounding-box regression targets.
It also adds gt_boxes to candidates and samples fg and bg rois for training.
Args:
rpn_rois: The proposed ROIs, e.g. from a region proposal network
scaled_gt_boxes: The ground truth boxes as (x1, y1, x2, y2, label). Coordinates are absolute pixels wrt. the input image.
num_classes: The number of classes in the data set
Returns:
rpn_target_rois - a set of rois containing the ground truth and a number of sampled fg and bg ROIs
label_targets - the target labels for the rois
bbox_targets - the regression coefficient targets for the rois
bbox_inside_weights - the weights for the regression loss
'''
ptl_param_string = "'num_classes': {}".format(num_classes)
ptl = user_function(ProposalTargetLayer(rpn_rois, scaled_gt_boxes, param_str=ptl_param_string))
rois = alias(ptl.outputs[0], name='rpn_target_rois')
label_targets = alias(ptl.outputs[1], name='label_targets')
bbox_targets = alias(ptl.outputs[2], name='bbox_targets')
bbox_inside_weights = alias(ptl.outputs[3], name='bbox_inside_w')
return rois, label_targets, bbox_targets, bbox_inside_weights
def create_proposal_target_layer(rpn_rois, scaled_gt_boxes, num_classes):
'''
Creates a proposal target layer that is used for training an object detection network as proposed in the "Faster R-CNN" paper:
Shaoqing Ren and Kaiming He and Ross Girshick and Jian Sun:
"Faster R-CNN: Towards Real-Time Object Detection with Region Proposal Networks"
Assigns object detection proposals to ground-truth targets.
Produces proposal classification labels and bounding-box regression targets.
It also adds gt_boxes to candidates and samples fg and bg rois for training.
Args:
rpn_rois: The proposed ROIs, e.g. from a region proposal network
scaled_gt_boxes: The ground truth boxes as (x1, y1, x2, y2, label). Coordinates are absolute pixels wrt. the input image.
num_classes: The number of classes in the data set
Returns:
rpn_target_rois - a set of rois containing the ground truth and a number of sampled fg and bg ROIs
label_targets - the target labels for the rois
bbox_targets - the regression coefficient targets for the rois
bbox_inside_weights - the weights for the regression loss
'''
ptl_param_string = "'num_classes': {}".format(num_classes)
ptl = user_function(
ProposalTargetLayer(rpn_rois,
scaled_gt_boxes,
param_str=ptl_param_string))
rois = alias(ptl.outputs[0], name='rpn_target_rois')
label_targets = alias(ptl.outputs[1], name='label_targets')
bbox_targets = alias(ptl.outputs[2], name='bbox_targets')
bbox_inside_weights = alias(ptl.outputs[3], name='bbox_inside_w')
return rois, label_targets, bbox_targets, bbox_inside_weights
def train_faster_rcnn_e2e(base_model_file_name, debug_output=False):
# Input variables denoting features and labeled ground truth rois (as 5-tuples per roi)
image_input = input_variable((num_channels, image_height, image_width), dynamic_axes=[Axis.default_batch_axis()], name=feature_node_name)
roi_input = input_variable((cfg["CNTK"].INPUT_ROIS_PER_IMAGE, 5), dynamic_axes=[Axis.default_batch_axis()])
dims_input = input_variable((6), dynamic_axes=[Axis.default_batch_axis()])
dims_node = alias(dims_input, name='dims_input')
# Instantiate the Faster R-CNN prediction model and loss function
loss, pred_error = create_faster_rcnn_predictor(base_model_file_name, image_input, roi_input, dims_node)
if debug_output:
print("Storing graphs and models to %s." % globalvars['output_path'])
plot(loss, os.path.join(globalvars['output_path'], "graph_frcn_train_e2e." + cfg["CNTK"].GRAPH_TYPE))
# Set learning parameters
e2e_lr_factor = globalvars['e2e_lr_factor']
e2e_lr_per_sample_scaled = [x * e2e_lr_factor for x in cfg["CNTK"].E2E_LR_PER_SAMPLE]
mm_schedule = momentum_schedule(cfg["CNTK"].MOMENTUM_PER_MB)
print("Using base model: {}".format(cfg["CNTK"].BASE_MODEL))
print("lr_per_sample: {}".format(e2e_lr_per_sample_scaled))
train_model(image_input, roi_input, dims_input, loss, pred_error,
e2e_lr_per_sample_scaled, mm_schedule, cfg["CNTK"].L2_REG_WEIGHT, globalvars['e2e_epochs'])
return create_eval_model(loss, image_input, dims_input)
def create_proposal_layer(rpn_cls_prob_reshape, rpn_bbox_pred, im_info, cfg, use_native_proposal_layer=False):
layer_config = {}
layer_config["feat_stride"] = cfg["MODEL"].FEATURE_STRIDE
layer_config["scales"] = cfg["DATA"].PROPOSAL_LAYER_SCALES
layer_config["train_pre_nms_topN"] = cfg["TRAIN"].RPN_PRE_NMS_TOP_N
layer_config["train_post_nms_topN"] = cfg["TRAIN"].RPN_POST_NMS_TOP_N
layer_config["train_nms_thresh"] = float(cfg["TRAIN"].RPN_NMS_THRESH)
layer_config["train_min_size"] = float(cfg["TRAIN"].RPN_MIN_SIZE)
layer_config["test_pre_nms_topN"] = cfg["TEST"].RPN_PRE_NMS_TOP_N
layer_config["test_post_nms_topN"] = cfg["TEST"].RPN_POST_NMS_TOP_N
layer_config["test_nms_thresh"] = float(cfg["TEST"].RPN_NMS_THRESH)
layer_config["test_min_size"] = float(cfg["TEST"].RPN_MIN_SIZE)
if use_native_proposal_layer:
cntk.ops.register_native_user_function('ProposalLayerOp',
'Cntk.ProposalLayerLib-' + cntk.__version__.rstrip('+'),
'CreateProposalLayer')
rpn_rois_raw = ops.native_user_function('ProposalLayerOp', [rpn_cls_prob_reshape, rpn_bbox_pred, im_info],
layer_config, 'native_proposal_layer')
else:
rpn_rois_raw = user_function(ProposalLayer(rpn_cls_prob_reshape, rpn_bbox_pred, im_info, layer_config))
return alias(rpn_rois_raw, name='rpn_rois')
def train_faster_rcnn_e2e(base_model_file_name, debug_output=False):
# Input variables denoting features and labeled ground truth rois (as 5-tuples per roi)
image_input = input_variable((num_channels, image_height, image_width), dynamic_axes=[Axis.default_batch_axis()], name=feature_node_name)
roi_input = input_variable((cfg["CNTK"].INPUT_ROIS_PER_IMAGE, 5), dynamic_axes=[Axis.default_batch_axis()])
dims_input = input_variable((6), dynamic_axes=[Axis.default_batch_axis()])
dims_node = alias(dims_input, name='dims_input')
# Instantiate the Faster R-CNN prediction model and loss function
loss, pred_error = create_faster_rcnn_predictor(base_model_file_name, image_input, roi_input, dims_node)
if debug_output:
print("Storing graphs and models to %s." % globalvars['output_path'])
plot(loss, os.path.join(globalvars['output_path'], "graph_frcn_train_e2e." + cfg["CNTK"].GRAPH_TYPE))
# Set learning parameters
e2e_lr_factor = globalvars['e2e_lr_factor']
e2e_lr_per_sample_scaled = [x * e2e_lr_factor for x in cfg["CNTK"].E2E_LR_PER_SAMPLE]
mm_schedule = momentum_schedule(cfg["CNTK"].MOMENTUM_PER_MB)
print("Using base model: {}".format(cfg["CNTK"].BASE_MODEL))
print("lr_per_sample: {}".format(e2e_lr_per_sample_scaled))
train_model(image_input, roi_input, dims_input, loss, pred_error,
e2e_lr_per_sample_scaled, mm_schedule, cfg["CNTK"].L2_REG_WEIGHT, globalvars['e2e_epochs'])
return create_eval_model(loss, image_input, dims_input)
def create_proposal_layer(rpn_cls_prob_reshape,
rpn_bbox_pred,
im_info,
cfg,
use_native_proposal_layer=False):
layer_config = {}
layer_config["feat_stride"] = cfg["MODEL"].FEATURE_STRIDE
layer_config["scales"] = cfg["DATA"].PROPOSAL_LAYER_SCALES
layer_config["train_pre_nms_topN"] = cfg["TRAIN"].RPN_PRE_NMS_TOP_N
layer_config["train_post_nms_topN"] = cfg["TRAIN"].RPN_POST_NMS_TOP_N
layer_config["train_nms_thresh"] = float(cfg["TRAIN"].RPN_NMS_THRESH)
layer_config["train_min_size"] = float(cfg["TRAIN"].RPN_MIN_SIZE)
layer_config["test_pre_nms_topN"] = cfg["TEST"].RPN_PRE_NMS_TOP_N
layer_config["test_post_nms_topN"] = cfg["TEST"].RPN_POST_NMS_TOP_N
layer_config["test_nms_thresh"] = float(cfg["TEST"].RPN_NMS_THRESH)
layer_config["test_min_size"] = float(cfg["TEST"].RPN_MIN_SIZE)
if use_native_proposal_layer:
cntk.ops.register_native_user_function(
'ProposalLayerOp',
'Cntk.ProposalLayerLib-' + cntk.__version__.rstrip('+'),
'CreateProposalLayer')
rpn_rois_raw = ops.native_user_function(
'ProposalLayerOp', [rpn_cls_prob_reshape, rpn_bbox_pred, im_info],
layer_config, 'native_proposal_layer')
else:
rpn_rois_raw = user_function(
ProposalLayer(rpn_cls_prob_reshape, rpn_bbox_pred, im_info,
layer_config))
return alias(rpn_rois_raw, name='rpn_rois')
def train_faster_rcnn_e2e(cfg):
# Input variables denoting features and labeled ground truth rois (as 5-tuples per roi)
image_input = input_variable(shape=(cfg.NUM_CHANNELS, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH),
dynamic_axes=[Axis.default_batch_axis()],
name=cfg["MODEL"].FEATURE_NODE_NAME)
roi_input = input_variable((cfg.INPUT_ROIS_PER_IMAGE, 5), dynamic_axes=[Axis.default_batch_axis()])
dims_input = input_variable((6), dynamic_axes=[Axis.default_batch_axis()])
dims_node = alias(dims_input, name='dims_input')
# Instantiate the Faster R-CNN prediction model and loss function
loss, pred_error = create_faster_rcnn_model(image_input, roi_input, dims_node, cfg)
if cfg["CNTK"].DEBUG_OUTPUT:
print("Storing graphs and models to %s." % cfg.OUTPUT_PATH)
plot(loss, os.path.join(cfg.OUTPUT_PATH, "graph_frcn_train_e2e." + cfg["CNTK"].GRAPH_TYPE))
# Set learning parameters
e2e_lr_factor = cfg["MODEL"].E2E_LR_FACTOR
e2e_lr_per_sample_scaled = [x * e2e_lr_factor for x in cfg["CNTK"].E2E_LR_PER_SAMPLE]
mm_schedule = momentum_schedule(cfg["CNTK"].MOMENTUM_PER_MB)
print("Using base model: {}".format(cfg["MODEL"].BASE_MODEL))
print("lr_per_sample: {}".format(e2e_lr_per_sample_scaled))
train_model(image_input, roi_input, dims_input, loss, pred_error,
e2e_lr_per_sample_scaled, mm_schedule, cfg["CNTK"].L2_REG_WEIGHT, cfg["CNTK"].E2E_MAX_EPOCHS, cfg)
return create_faster_rcnn_eval_model(loss, image_input, dims_input, cfg)
def train_faster_rcnn_e2e(cfg):
# Input variables denoting features and labeled ground truth rois (as 5-tuples per roi)
image_input = input_variable(shape=(cfg.NUM_CHANNELS, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH),
dynamic_axes=[Axis.default_batch_axis()],
name=cfg["MODEL"].FEATURE_NODE_NAME)
roi_input = input_variable((cfg.INPUT_ROIS_PER_IMAGE, 5), dynamic_axes=[Axis.default_batch_axis()])
dims_input = input_variable((6), dynamic_axes=[Axis.default_batch_axis()])
dims_node = alias(dims_input, name='dims_input')
# Instantiate the Faster R-CNN prediction model and loss function
loss, pred_error = create_faster_rcnn_model(image_input, roi_input, dims_node, cfg)
if cfg["CNTK"].DEBUG_OUTPUT:
print("Storing graphs and models to %s." % cfg.OUTPUT_PATH)
plot(loss, os.path.join(cfg.OUTPUT_PATH, "graph_frcn_train_e2e." + cfg["CNTK"].GRAPH_TYPE))
# Set learning parameters
e2e_lr_factor = cfg["MODEL"].E2E_LR_FACTOR
e2e_lr_per_sample_scaled = [x * e2e_lr_factor for x in cfg["CNTK"].E2E_LR_PER_SAMPLE]
mm_schedule = momentum_schedule(cfg["CNTK"].MOMENTUM_PER_MB)
print("Using base model: {}".format(cfg["MODEL"].BASE_MODEL))
print("lr_per_sample: {}".format(e2e_lr_per_sample_scaled))
train_model(image_input, roi_input, dims_input, loss, pred_error,
e2e_lr_per_sample_scaled, mm_schedule, cfg["CNTK"].L2_REG_WEIGHT, cfg["CNTK"].E2E_MAX_EPOCHS, cfg)
return create_faster_rcnn_eval_model(loss, image_input, dims_input, cfg)
def create_network(cfg):
"""build the network for faster rcnn"""
##create input variables
features = C.input_variable(shape=(cfg.NUM_CHANNELS, cfg.IMAGE_HEIGHT,
cfg.IMAGE_WIDTH),
dynamic_axes=[C.Axis.default_batch_axis()],
name=cfg["MODEL"].FEATURE_NODE_NAME)
##roi_input
scaled_gt_boxes = C.input_variable(
(cfg.INPUT_ROIS_PER_IMAGE, 5),
dynamic_axes=[C.Axis.default_batch_axis()])
dims_in = C.input_variable((6), dynamic_axes=[C.Axis.default_batch_axis()])
dims_input = C.alias(dims_in, name='dims_input')
# Load the pre-trained classification net and clone layers
base_model = C.load_model(cfg['BASE_MODEL_PATH'])
conv_layers = clone_conv_layers(base_model, cfg)
fc_layers = clone_model(base_model, [cfg["MODEL"].POOL_NODE_NAME],
[cfg["MODEL"].LAST_HIDDEN_NODE_NAME],
clone_method=CloneMethod.clone)
# Normalization and conv layers
feat_norm = features - C.Constant([[[v]]
for v in cfg["MODEL"].IMG_PAD_COLOR])
conv_out = conv_layers(feat_norm)
# RPN and prediction targets
rpn_rois, rpn_losses = create_rpn(conv_out, scaled_gt_boxes, dims_input,
cfg)
rois, label_targets, bbox_targets, bbox_inside_weights = create_proposal_target_layer(
rpn_rois, scaled_gt_boxes, cfg)
# Fast RCNN and losses
cls_score, bbox_pred = create_fast_rcnn_predictor(conv_out, rois,
fc_layers, cfg)
detection_losses = create_detection_losses(cls_score, label_targets,
bbox_pred, rois, bbox_targets,
bbox_inside_weights, cfg)
loss = rpn_losses + detection_losses
pred_error = classification_error(cls_score, label_targets, axis=1)
e2e_lr_factor = cfg["MODEL"].E2E_LR_FACTOR
e2e_lr_per_sample_scaled = [
x * e2e_lr_factor for x in cfg["CNTK"].E2E_LR_PER_SAMPLE
]
mm_schedule = momentum_schedule(cfg["CNTK"].MOMENTUM_PER_MB)
print("Using base model: {}".format(cfg["MODEL"].BASE_MODEL))
print("lr_per_sample: {}".format(e2e_lr_per_sample_scaled))
return {
'features': features,
'roi_input': scaled_gt_boxes,
'loss': loss,
'pred_error': pred_error,
'dim_input': dims_in
}
def conv_from_weights(x, weights, bias=None, padding=True, name=""):
""" weights is a numpy array """
k = C.parameter(shape=weights.shape, init=weights)
y = C.convolution(k, x, auto_padding=[False, padding, padding])
if bias:
b = C.parameter(shape=bias.shape, init=bias)
y = y + bias
y = C.alias(y, name=name)
return y
def _inject_name(f, name):
'''
Call this at the end of any layer or block that takes an optional name argument.
'''
if name:
if len(f.outputs) == 1:
f = alias(f, name=name)
else:
f = combine(list(f.outputs), name=name) # BUGBUG: Does this actually name things?
return f
def _inject_name(f, name):
'''
Call this at the end of any layer or block that takes an optional name argument.
'''
if name:
if len(f.outputs) == 1:
f = alias(f, name=name)
else:
f = combine(list(f.outputs), name=name) # BUGBUG: Does this actually name things?
return f
def create_proposal_target_layer(rpn_rois, scaled_gt_boxes, cfg):
'''
Creates a proposal target layer that is used for training an object detection network as proposed in the "Faster R-CNN" paper:
Shaoqing Ren and Kaiming He and Ross Girshick and Jian Sun:
"Faster R-CNN: Towards Real-Time Object Detection with Region Proposal Networks"
Assigns object detection proposals to ground-truth targets.
Produces proposal classification labels and bounding-box regression targets.
It also adds gt_boxes to candidates and samples fg and bg rois for training.
Args:
rpn_rois: The proposed ROIs, e.g. from a region proposal network
scaled_gt_boxes: The ground truth boxes as (x1, y1, x2, y2, label). Coordinates are absolute pixels wrt. the input image.
num_classes: The number of classes in the data set
Returns:
rpn_target_rois - a set of rois containing the ground truth and a number of sampled fg and bg ROIs
label_targets - the target labels for the rois
bbox_targets - the regression coefficient targets for the rois
bbox_inside_weights - the weights for the regression loss
'''
ptl_param_string = "'num_classes': {}".format(cfg["DATA"].NUM_CLASSES)
ptl = user_function(
ProposalTargetLayer(rpn_rois,
scaled_gt_boxes,
batch_size=cfg.NUM_ROI_PROPOSALS,
fg_fraction=cfg["TRAIN"].FG_FRACTION,
normalize_targets=cfg.BBOX_NORMALIZE_TARGETS,
normalize_means=cfg.BBOX_NORMALIZE_MEANS,
normalize_stds=cfg.BBOX_NORMALIZE_STDS,
fg_thresh=cfg["TRAIN"].FG_THRESH,
bg_thresh_hi=cfg["TRAIN"].BG_THRESH_HI,
bg_thresh_lo=cfg["TRAIN"].BG_THRESH_LO,
param_str=ptl_param_string))
# use an alias if you need to access the outputs, e.g., when cloning a trained network
rois = alias(ptl.outputs[0], name='rpn_target_rois')
label_targets = ptl.outputs[1]
bbox_targets = ptl.outputs[2]
bbox_inside_weights = ptl.outputs[3]
return rois, label_targets, bbox_targets, bbox_inside_weights
def create_proposal_target_layer(rpn_rois, scaled_gt_boxes, cfg):
'''
Creates a proposal target layer that is used for training an object detection network as proposed in the "Faster R-CNN" paper:
Shaoqing Ren and Kaiming He and Ross Girshick and Jian Sun:
"Faster R-CNN: Towards Real-Time Object Detection with Region Proposal Networks"
Assigns object detection proposals to ground-truth targets.
Produces proposal classification labels and bounding-box regression targets.
It also adds gt_boxes to candidates and samples fg and bg rois for training.
Args:
rpn_rois: The proposed ROIs, e.g. from a region proposal network
scaled_gt_boxes: The ground truth boxes as (x1, y1, x2, y2, label). Coordinates are absolute pixels wrt. the input image.
num_classes: The number of classes in the data set
Returns:
rpn_target_rois - a set of rois containing the ground truth and a number of sampled fg and bg ROIs
label_targets - the target labels for the rois
bbox_targets - the regression coefficient targets for the rois
bbox_inside_weights - the weights for the regression loss
'''
ptl_param_string = "'num_classes': {}".format(cfg["DATA"].NUM_CLASSES)
ptl = user_function(ProposalTargetLayer(rpn_rois, scaled_gt_boxes,
batch_size=cfg.NUM_ROI_PROPOSALS,
fg_fraction=cfg["TRAIN"].FG_FRACTION,
normalize_targets=cfg.BBOX_NORMALIZE_TARGETS,
normalize_means=cfg.BBOX_NORMALIZE_MEANS,
normalize_stds=cfg.BBOX_NORMALIZE_STDS,
fg_thresh=cfg["TRAIN"].FG_THRESH,
bg_thresh_hi=cfg["TRAIN"].BG_THRESH_HI,
bg_thresh_lo=cfg["TRAIN"].BG_THRESH_LO,
param_str=ptl_param_string))
# use an alias if you need to access the outputs, e.g., when cloning a trained network
rois = alias(ptl.outputs[0], name='rpn_target_rois')
label_targets = ptl.outputs[1]
bbox_targets = ptl.outputs[2]
bbox_inside_weights = ptl.outputs[3]
return rois, label_targets, bbox_targets, bbox_inside_weights
def test_outputs_passing():
in1 = C.input_variable(shape=(1,))
a = C.alias(in1 + 1, name='a')
b = a + 2
expected = [[2], [3]]
result = b.eval({in1: [[1], [2]]}, outputs=a.outputs)
assert np.array_equal(result, expected)
result = b.eval({in1: [[1], [2]]}, outputs=list(a.outputs))
assert np.array_equal(result, expected)
result = b.eval({in1: [[1], [2]]}, outputs=a.output)
assert np.array_equal(result, expected)
result = b.eval({in1: [[1], [2]]}, outputs=a)
assert np.array_equal(result, expected)
with pytest.raises(TypeError):
b.eval({in1: [[1], [2]]}, outputs=[a.outputs])
def test_outputs_passing():
in1 = C.input_variable(shape=(1, ))
a = C.alias(in1 + 1, name='a')
b = a + 2
expected = [[2], [3]]
result = b.eval({in1: [[1], [2]]}, outputs=a.outputs)
assert np.array_equal(result, expected)
result = b.eval({in1: [[1], [2]]}, outputs=list(a.outputs))
assert np.array_equal(result, expected)
result = b.eval({in1: [[1], [2]]}, outputs=a.output)
assert np.array_equal(result, expected)
result = b.eval({in1: [[1], [2]]}, outputs=a)
assert np.array_equal(result, expected)
with pytest.raises(TypeError):
b.eval({in1: [[1], [2]]}, outputs=[a.outputs])
def block(x):
self_attended = mha_block(x, C.alias(x), C.alias(x))
hidden = feed_foward(self_attended)
output = layernorm(hidden + self_attended) # residual connection
return output
def train_faster_rcnn_alternating(base_model_file_name, debug_output=False):
'''
4-Step Alternating Training scheme from the Faster R-CNN paper:
# Create initial network, only rpn, without detection network
# --> train only the rpn (and conv3_1 and up for VGG16)
# buffer region proposals from rpn
# Create full network, initialize conv layers with imagenet, use buffered proposals
# --> train only detection network (and conv3_1 and up for VGG16)
# Keep conv weights from detection network and fix them
# --> train only rpn
# buffer region proposals from rpn
# Keep conv and rpn weights from step 3 and fix them
# --> train only detection network
'''
# Learning parameters
rpn_lr_factor = globalvars['rpn_lr_factor']
rpn_lr_per_sample_scaled = [
x * rpn_lr_factor for x in cfg["CNTK"].RPN_LR_PER_SAMPLE
]
frcn_lr_factor = globalvars['frcn_lr_factor']
frcn_lr_per_sample_scaled = [
x * frcn_lr_factor for x in cfg["CNTK"].FRCN_LR_PER_SAMPLE
]
l2_reg_weight = cfg["CNTK"].L2_REG_WEIGHT
mm_schedule = momentum_schedule(globalvars['momentum_per_mb'])
rpn_epochs = globalvars['rpn_epochs']
frcn_epochs = globalvars['frcn_epochs']
print("Using base model: {}".format(cfg["CNTK"].BASE_MODEL))
print("rpn_lr_per_sample: {}".format(rpn_lr_per_sample_scaled))
print("frcn_lr_per_sample: {}".format(frcn_lr_per_sample_scaled))
if debug_output:
print("Storing graphs and models to %s." % globalvars['output_path'])
# Input variables denoting features, labeled ground truth rois (as 5-tuples per roi) and image dimensions
image_input = input_variable((num_channels, image_height, image_width),
dynamic_axes=[Axis.default_batch_axis()],
name=feature_node_name)
feat_norm = image_input - normalization_const
roi_input = input_variable((cfg["CNTK"].INPUT_ROIS_PER_IMAGE, 5),
dynamic_axes=[Axis.default_batch_axis()])
scaled_gt_boxes = alias(roi_input, name='roi_input')
dims_input = input_variable((6), dynamic_axes=[Axis.default_batch_axis()])
dims_node = alias(dims_input, name='dims_input')
rpn_rois_input = input_variable((cfg["TRAIN"].RPN_POST_NMS_TOP_N, 4),
dynamic_axes=[Axis.default_batch_axis()])
rpn_rois_buf = alias(rpn_rois_input, name='rpn_rois')
# base image classification model (e.g. VGG16 or AlexNet)
base_model = load_model(base_model_file_name)
print("stage 1a - rpn")
if True:
# Create initial network, only rpn, without detection network
# initial weights train?
# conv: base_model only conv3_1 and up
# rpn: init new yes
# frcn: - -
# conv layers
conv_layers = clone_conv_layers(base_model)
conv_out = conv_layers(feat_norm)
# RPN and losses
rpn_rois, rpn_losses = create_rpn(
conv_out,
scaled_gt_boxes,
dims_node,
proposal_layer_param_string=cfg["CNTK"].PROPOSAL_LAYER_PARAMS)
stage1_rpn_network = combine([rpn_rois, rpn_losses])
# train
if debug_output:
plot(
stage1_rpn_network,
os.path.join(
globalvars['output_path'],
"graph_frcn_train_stage1a_rpn." + cfg["CNTK"].GRAPH_TYPE))
train_model(image_input,
roi_input,
dims_input,
rpn_losses,
rpn_losses,
rpn_lr_per_sample_scaled,
mm_schedule,
l2_reg_weight,
epochs_to_train=rpn_epochs)
print("stage 1a - buffering rpn proposals")
buffered_proposals_s1 = compute_rpn_proposals(stage1_rpn_network,
image_input, roi_input,
dims_input)
print("stage 1b - frcn")
if True:
# Create full network, initialize conv layers with imagenet, fix rpn weights
# initial weights train?
# conv: base_model only conv3_1 and up
# rpn: stage1a rpn model no --> use buffered proposals
# frcn: base_model + new yes
# conv_layers
conv_layers = clone_conv_layers(base_model)
conv_out = conv_layers(feat_norm)
# use buffered proposals in target layer
rois, label_targets, bbox_targets, bbox_inside_weights = \
create_proposal_target_layer(rpn_rois_buf, scaled_gt_boxes, num_classes=globalvars['num_classes'])
# Fast RCNN and losses
fc_layers = clone_model(base_model, [pool_node_name],
[last_hidden_node_name], CloneMethod.clone)
cls_score, bbox_pred = create_fast_rcnn_predictor(
conv_out, rois, fc_layers)
detection_losses = create_detection_losses(cls_score, label_targets,
rois, bbox_pred,
bbox_targets,
bbox_inside_weights)
pred_error = classification_error(cls_score,
label_targets,
axis=1,
name="pred_error")
stage1_frcn_network = combine(
[rois, cls_score, bbox_pred, detection_losses, pred_error])
# train
if debug_output:
plot(
stage1_frcn_network,
os.path.join(
globalvars['output_path'],
"graph_frcn_train_stage1b_frcn." + cfg["CNTK"].GRAPH_TYPE))
train_model(image_input,
roi_input,
dims_input,
detection_losses,
pred_error,
frcn_lr_per_sample_scaled,
mm_schedule,
l2_reg_weight,
epochs_to_train=frcn_epochs,
rpn_rois_input=rpn_rois_input,
buffered_rpn_proposals=buffered_proposals_s1)
buffered_proposals_s1 = None
print("stage 2a - rpn")
if True:
# Keep conv weights from detection network and fix them
# initial weights train?
# conv: stage1b frcn model no
# rpn: stage1a rpn model yes
# frcn: - -
# conv_layers
conv_layers = clone_model(stage1_frcn_network, [feature_node_name],
[last_conv_node_name], CloneMethod.freeze)
conv_out = conv_layers(image_input)
# RPN and losses
rpn = clone_model(stage1_rpn_network,
[last_conv_node_name, "roi_input", "dims_input"],
["rpn_rois", "rpn_losses"], CloneMethod.clone)
rpn_net = rpn(conv_out, dims_node, scaled_gt_boxes)
rpn_rois = rpn_net.outputs[0]
rpn_losses = rpn_net.outputs[1]
stage2_rpn_network = combine([rpn_rois, rpn_losses])
# train
if debug_output:
plot(
stage2_rpn_network,
os.path.join(
globalvars['output_path'],
"graph_frcn_train_stage2a_rpn." + cfg["CNTK"].GRAPH_TYPE))
train_model(image_input,
roi_input,
dims_input,
rpn_losses,
rpn_losses,
rpn_lr_per_sample_scaled,
mm_schedule,
l2_reg_weight,
epochs_to_train=rpn_epochs)
print("stage 2a - buffering rpn proposals")
buffered_proposals_s2 = compute_rpn_proposals(stage2_rpn_network,
image_input, roi_input,
dims_input)
print("stage 2b - frcn")
if True:
# Keep conv and rpn weights from step 3 and fix them
# initial weights train?
# conv: stage2a rpn model no
# rpn: stage2a rpn model no --> use buffered proposals
# frcn: stage1b frcn model yes -
# conv_layers
conv_layers = clone_model(stage2_rpn_network, [feature_node_name],
[last_conv_node_name], CloneMethod.freeze)
conv_out = conv_layers(image_input)
# Fast RCNN and losses
frcn = clone_model(stage1_frcn_network,
[last_conv_node_name, "rpn_rois", "roi_input"], [
"cls_score", "bbox_regr", "rpn_target_rois",
"detection_losses", "pred_error"
], CloneMethod.clone)
stage2_frcn_network = frcn(conv_out, rpn_rois_buf, scaled_gt_boxes)
detection_losses = stage2_frcn_network.outputs[3]
pred_error = stage2_frcn_network.outputs[4]
# train
if debug_output:
plot(
stage2_frcn_network,
os.path.join(
globalvars['output_path'],
"graph_frcn_train_stage2b_frcn." + cfg["CNTK"].GRAPH_TYPE))
train_model(image_input,
roi_input,
dims_input,
detection_losses,
pred_error,
frcn_lr_per_sample_scaled,
mm_schedule,
l2_reg_weight,
epochs_to_train=frcn_epochs,
rpn_rois_input=rpn_rois_input,
buffered_rpn_proposals=buffered_proposals_s2)
buffered_proposals_s2 = None
return create_eval_model(stage2_frcn_network,
image_input,
dims_input,
rpn_model=stage2_rpn_network)
def create_rpn(conv_out, scaled_gt_boxes, im_info, add_loss_functions=True,
proposal_layer_param_string=None):
'''
Creates a region proposal network for object detection as proposed in the "Faster R-CNN" paper:
Shaoqing Ren and Kaiming He and Ross Girshick and Jian Sun:
"Faster R-CNN: Towards Real-Time Object Detection with Region Proposal Networks"
Outputs object detection proposals by applying estimated bounding-box
transformations to a set of regular boxes (called "anchors").
Args:
conv_out: The convolutional feature map, i.e. the output of the conv layers from the pretrained classification network
scaled_gt_boxes: The ground truth boxes as (x1, y1, x2, y2, label). Coordinates are absolute pixels wrt. the input image.
im_info: (image_widht, image_height, image_scale) as CNTK variable or constant
add_loss_functions: If set to True rpn_losses will be returned, otherwise None is returned for the losses
proposal_layer_param_string: A yaml parameter string that is passed to the proposal layer.
Returns:
rpn_rois - the proposed ROIs
rpn_losses - the losses (SmoothL1 loss for bbox regression plus cross entropy for objectness)
'''
# RPN network
# init = 'normal', initValueScale = 0.01, initBias = 0.1
rpn_conv_3x3 = Convolution((3, 3), 256, activation=relu, pad=True, strides=1,
init = normal(scale=0.01), init_bias=0.1)(conv_out)
rpn_cls_score = Convolution((1, 1), 18, activation=None, name="rpn_cls_score",
init = normal(scale=0.01), init_bias=0.1)(rpn_conv_3x3) # 2(bg/fg) * 9(anchors)
rpn_bbox_pred = Convolution((1, 1), 36, activation=None, name="rpn_bbox_pred",
init = normal(scale=0.01), init_bias=0.1)(rpn_conv_3x3) # 4(coords) * 9(anchors)
# apply softmax to get (bg, fg) probabilities and reshape predictions back to grid of (18, H, W)
num_predictions = int(np.prod(rpn_cls_score.shape) / 2)
rpn_cls_score_rshp = reshape(rpn_cls_score, (2, num_predictions))
rpn_cls_prob = softmax(rpn_cls_score_rshp, axis=0, name="objness_softmax")
rpn_cls_prob_reshape = reshape(rpn_cls_prob, rpn_cls_score.shape)
# proposal layer
rpn_rois_raw = user_function(ProposalLayer(rpn_cls_prob_reshape, rpn_bbox_pred, im_info, param_str=proposal_layer_param_string))
rpn_rois = alias(rpn_rois_raw, name='rpn_rois')
rpn_losses = None
if(add_loss_functions):
# RPN targets
# Comment: rpn_cls_score is only passed vvv to get width and height of the conv feature map ...
atl = user_function(AnchorTargetLayer(rpn_cls_score, scaled_gt_boxes, im_info, param_str=proposal_layer_param_string))
rpn_labels = atl.outputs[0]
rpn_bbox_targets = atl.outputs[1]
rpn_bbox_inside_weights = atl.outputs[2]
# For loss functions: ignore label predictions for the 'ignore label',
# i.e. set target and prediction to 0 --> needs to be softmaxed before
rpn_labels_rshp = reshape(rpn_labels, (1, num_predictions))
ignore = user_function(IgnoreLabel(rpn_cls_prob, rpn_labels_rshp, ignore_label=-1))
rpn_cls_prob_ignore = ignore.outputs[0]
fg_targets = ignore.outputs[1]
bg_targets = 1 - fg_targets
rpn_labels_ignore = splice(bg_targets, fg_targets, axis=0)
# RPN losses
rpn_loss_cls = cross_entropy_with_softmax(rpn_cls_prob_ignore, rpn_labels_ignore, axis=0)
rpn_loss_bbox = user_function(SmoothL1Loss(rpn_bbox_pred, rpn_bbox_targets, rpn_bbox_inside_weights))
rpn_losses = plus(reduce_sum(rpn_loss_cls), reduce_sum(rpn_loss_bbox), name="rpn_losses")
return rpn_rois, rpn_losses
def create_rpn(conv_out,
scaled_gt_boxes,
im_info,
add_loss_functions=True,
proposal_layer_param_string=None):
'''
Creates a region proposal network for object detection as proposed in the "Faster R-CNN" paper:
Shaoqing Ren and Kaiming He and Ross Girshick and Jian Sun:
"Faster R-CNN: Towards Real-Time Object Detection with Region Proposal Networks"
Outputs object detection proposals by applying estimated bounding-box
transformations to a set of regular boxes (called "anchors").
Args:
conv_out: The convolutional feature map, i.e. the output of the conv layers from the pretrained classification network
scaled_gt_boxes: The ground truth boxes as (x1, y1, x2, y2, label). Coordinates are absolute pixels wrt. the input image.
im_info: (image_widht, image_height, image_scale) as CNTK variable or constant
add_loss_functions: If set to True rpn_losses will be returned, otherwise None is returned for the losses
proposal_layer_param_string: A yaml parameter string that is passed to the proposal layer.
Returns:
rpn_rois - the proposed ROIs
rpn_losses - the losses (SmoothL1 loss for bbox regression plus cross entropy for objectness)
'''
# RPN network
# init = 'normal', initValueScale = 0.01, initBias = 0.1
rpn_conv_3x3 = Convolution((3, 3),
256,
activation=relu,
pad=True,
strides=1,
init=normal(scale=0.01),
init_bias=0.1)(conv_out)
rpn_cls_score = Convolution(
(1, 1),
18,
activation=None,
name="rpn_cls_score",
init=normal(scale=0.01),
init_bias=0.1)(rpn_conv_3x3) # 2(bg/fg) * 9(anchors)
rpn_bbox_pred = Convolution(
(1, 1),
36,
activation=None,
name="rpn_bbox_pred",
init=normal(scale=0.01),
init_bias=0.1)(rpn_conv_3x3) # 4(coords) * 9(anchors)
# apply softmax to get (bg, fg) probabilities and reshape predictions back to grid of (18, H, W)
num_predictions = int(np.prod(rpn_cls_score.shape) / 2)
rpn_cls_score_rshp = reshape(rpn_cls_score, (2, num_predictions))
rpn_cls_prob = softmax(rpn_cls_score_rshp, axis=0, name="objness_softmax")
rpn_cls_prob_reshape = reshape(rpn_cls_prob, rpn_cls_score.shape)
# proposal layer
rpn_rois_raw = user_function(
ProposalLayer(rpn_cls_prob_reshape,
rpn_bbox_pred,
im_info,
param_str=proposal_layer_param_string))
rpn_rois = alias(rpn_rois_raw, name='rpn_rois')
rpn_losses = None
if (add_loss_functions):
# RPN targets
# Comment: rpn_cls_score is only passed vvv to get width and height of the conv feature map ...
atl = user_function(
AnchorTargetLayer(rpn_cls_score,
scaled_gt_boxes,
im_info,
param_str=proposal_layer_param_string))
rpn_labels = atl.outputs[0]
rpn_bbox_targets = atl.outputs[1]
rpn_bbox_inside_weights = atl.outputs[2]
# For loss functions: ignore label predictions for the 'ignore label',
# i.e. set target and prediction to 0 --> needs to be softmaxed before
rpn_labels_rshp = reshape(rpn_labels, (1, num_predictions))
ignore = user_function(
IgnoreLabel(rpn_cls_prob, rpn_labels_rshp, ignore_label=-1))
rpn_cls_prob_ignore = ignore.outputs[0]
fg_targets = ignore.outputs[1]
bg_targets = 1 - fg_targets
rpn_labels_ignore = splice(bg_targets, fg_targets, axis=0)
# RPN losses
rpn_loss_cls = cross_entropy_with_softmax(rpn_cls_prob_ignore,
rpn_labels_ignore,
axis=0)
rpn_loss_bbox = user_function(
SmoothL1Loss(rpn_bbox_pred, rpn_bbox_targets,
rpn_bbox_inside_weights))
rpn_losses = plus(reduce_sum(rpn_loss_cls),
reduce_sum(rpn_loss_bbox),
name="rpn_losses")
return rpn_rois, rpn_losses
def create_rpn(conv_out, scaled_gt_boxes, im_info, add_loss_functions=True,
proposal_layer_param_string=None, conv_bias_init=0.0):
'''
Creates a region proposal network for object detection as proposed in the "Faster R-CNN" paper:
Shaoqing Ren and Kaiming He and Ross Girshick and Jian Sun:
"Faster R-CNN: Towards Real-Time Object Detection with Region Proposal Networks"
Outputs object detection proposals by applying estimated bounding-box
transformations to a set of regular boxes (called "anchors").
Args:
conv_out: The convolutional feature map, i.e. the output of the conv layers from the pretrained classification network
scaled_gt_boxes: The ground truth boxes as (x1, y1, x2, y2, label). Coordinates are absolute pixels wrt. the input image.
im_info: A CNTK variable or constant containing
(pad_width, pad_height, scaled_image_width, scaled_image_height, orig_img_width, orig_img_height)
e.g. (1000, 1000, 1000, 600, 500, 300) for an original image of 600x300 that is scaled and padded to 1000x1000
add_loss_functions: If set to True rpn_losses will be returned, otherwise None is returned for the losses
proposal_layer_param_string: A yaml parameter string that is passed to the proposal layer.
Returns:
rpn_rois - the proposed ROIs
rpn_losses - the losses (SmoothL1 loss for bbox regression plus cross entropy for objectness)
'''
# RPN network
# init = 'normal', initValueScale = 0.01, initBias = 0.1
num_channels = cfg["CNTK"].RPN_NUM_CHANNELS
rpn_conv_3x3 = Convolution((3, 3), num_channels, activation=relu, pad=True, strides=1,
init = normal(scale=0.01), init_bias=conv_bias_init)(conv_out)
rpn_cls_score = Convolution((1, 1), 18, activation=None, name="rpn_cls_score",
init = normal(scale=0.01), init_bias=conv_bias_init)(rpn_conv_3x3) # 2(bg/fg) * 9(anchors)
rpn_bbox_pred = Convolution((1, 1), 36, activation=None, name="rpn_bbox_pred",
init = normal(scale=0.01), init_bias=conv_bias_init)(rpn_conv_3x3) # 4(coords) * 9(anchors)
# apply softmax to get (bg, fg) probabilities and reshape predictions back to grid of (18, H, W)
num_predictions = int(rpn_cls_score.shape[0] / 2)
rpn_cls_score_rshp = reshape(rpn_cls_score, (2, num_predictions, rpn_cls_score.shape[1], rpn_cls_score.shape[2]), name="rpn_cls_score_rshp")
p_rpn_cls_score_rshp = cntk.placeholder()
rpn_cls_sm = softmax(p_rpn_cls_score_rshp, axis=0)
rpn_cls_prob = cntk.as_block(rpn_cls_sm, [(p_rpn_cls_score_rshp, rpn_cls_score_rshp)], 'Softmax', 'rpn_cls_prob')
rpn_cls_prob_reshape = reshape(rpn_cls_prob, rpn_cls_score.shape, name="rpn_cls_prob_reshape")
# proposal layer
rpn_rois_raw = user_function(ProposalLayer(rpn_cls_prob_reshape, rpn_bbox_pred, im_info, param_str=proposal_layer_param_string))
rpn_rois = alias(rpn_rois_raw, name='rpn_rois')
rpn_losses = None
if(add_loss_functions):
# RPN targets
# Comment: rpn_cls_score is only passed vvv to get width and height of the conv feature map ...
atl = user_function(AnchorTargetLayer(rpn_cls_score, scaled_gt_boxes, im_info, param_str=proposal_layer_param_string))
rpn_labels = atl.outputs[0]
rpn_bbox_targets = atl.outputs[1]
rpn_bbox_inside_weights = atl.outputs[2]
# classification loss
p_rpn_labels = cntk.placeholder()
p_rpn_cls_score_rshp = cntk.placeholder()
keeps = cntk.greater_equal(p_rpn_labels, 0.0)
fg_labels = element_times(p_rpn_labels, keeps, name="fg_targets")
bg_labels = minus(1, fg_labels, name="bg_targets")
rpn_labels_ignore = splice(bg_labels, fg_labels, axis=0)
rpn_ce = cross_entropy_with_softmax(p_rpn_cls_score_rshp, rpn_labels_ignore, axis=0)
rpn_loss_cls = element_times(rpn_ce, keeps)
# The terms that are accounted for in the cls loss are those that have a label >= 0
cls_num_terms = reduce_sum(keeps)
cls_normalization_factor = 1.0 / cls_num_terms
normalized_rpn_cls_loss = reduce_sum(rpn_loss_cls) * cls_normalization_factor
reduced_rpn_loss_cls = cntk.as_block(normalized_rpn_cls_loss,
[(p_rpn_labels, rpn_labels), (p_rpn_cls_score_rshp, rpn_cls_score_rshp)],
'CE_with_ignore', 'norm_rpn_cls_loss')
# regression loss
p_rpn_bbox_pred = cntk.placeholder()
p_rpn_bbox_targets = cntk.placeholder()
p_rpn_bbox_inside_weights = cntk.placeholder()
rpn_loss_bbox = SmoothL1Loss(cfg["CNTK"].SIGMA_RPN_L1, p_rpn_bbox_pred, p_rpn_bbox_targets, p_rpn_bbox_inside_weights, 1.0)
# The bbox loss is normalized by the rpn batch size
bbox_normalization_factor = 1.0 / cfg["TRAIN"].RPN_BATCHSIZE
normalized_rpn_bbox_loss = reduce_sum(rpn_loss_bbox) * bbox_normalization_factor
reduced_rpn_loss_bbox = cntk.as_block(normalized_rpn_bbox_loss,
[(p_rpn_bbox_pred, rpn_bbox_pred), (p_rpn_bbox_targets, rpn_bbox_targets),
(p_rpn_bbox_inside_weights, rpn_bbox_inside_weights)],
'SmoothL1Loss', 'norm_rpn_bbox_loss')
rpn_losses = plus(reduced_rpn_loss_cls, reduced_rpn_loss_bbox, name="rpn_losses")
return rpn_rois, rpn_losses
def train_faster_rcnn_alternating(cfg):
'''
4-Step Alternating Training scheme from the Faster R-CNN paper:
# Create initial network, only rpn, without detection network
# --> train only the rpn (and conv3_1 and up for VGG16)
# buffer region proposals from rpn
# Create full network, initialize conv layers with imagenet, use buffered proposals
# --> train only detection network (and conv3_1 and up for VGG16)
# Keep conv weights from detection network and fix them
# --> train only rpn
# buffer region proposals from rpn
# Keep conv and rpn weights from step 3 and fix them
# --> train only detection network
'''
# setting pre- and post-nms top N to training values since buffered proposals are used for further training
test_pre = cfg["TEST"].RPN_PRE_NMS_TOP_N
test_post = cfg["TEST"].RPN_POST_NMS_TOP_N
cfg["TEST"].RPN_PRE_NMS_TOP_N = cfg["TRAIN"].RPN_PRE_NMS_TOP_N
cfg["TEST"].RPN_POST_NMS_TOP_N = cfg["TRAIN"].RPN_POST_NMS_TOP_N
# Learning parameters
rpn_lr_factor = cfg["MODEL"].RPN_LR_FACTOR
rpn_lr_per_sample_scaled = [x * rpn_lr_factor for x in cfg["CNTK"].RPN_LR_PER_SAMPLE]
frcn_lr_factor = cfg["MODEL"].FRCN_LR_FACTOR
frcn_lr_per_sample_scaled = [x * frcn_lr_factor for x in cfg["CNTK"].FRCN_LR_PER_SAMPLE]
l2_reg_weight = cfg["CNTK"].L2_REG_WEIGHT
mm_schedule = momentum_schedule(cfg["CNTK"].MOMENTUM_PER_MB)
rpn_epochs = cfg["CNTK"].RPN_EPOCHS
frcn_epochs = cfg["CNTK"].FRCN_EPOCHS
feature_node_name = cfg["MODEL"].FEATURE_NODE_NAME
last_conv_node_name = cfg["MODEL"].LAST_CONV_NODE_NAME
print("Using base model: {}".format(cfg["MODEL"].BASE_MODEL))
print("rpn_lr_per_sample: {}".format(rpn_lr_per_sample_scaled))
print("frcn_lr_per_sample: {}".format(frcn_lr_per_sample_scaled))
debug_output=cfg["CNTK"].DEBUG_OUTPUT
if debug_output:
print("Storing graphs and models to %s." % cfg.OUTPUT_PATH)
# Input variables denoting features, labeled ground truth rois (as 5-tuples per roi) and image dimensions
image_input = input_variable(shape=(cfg.NUM_CHANNELS, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH),
dynamic_axes=[Axis.default_batch_axis()],
name=feature_node_name)
feat_norm = image_input - Constant([[[v]] for v in cfg["MODEL"].IMG_PAD_COLOR])
roi_input = input_variable((cfg.INPUT_ROIS_PER_IMAGE, 5), dynamic_axes=[Axis.default_batch_axis()])
scaled_gt_boxes = alias(roi_input, name='roi_input')
dims_input = input_variable((6), dynamic_axes=[Axis.default_batch_axis()])
dims_node = alias(dims_input, name='dims_input')
rpn_rois_input = input_variable((cfg["TRAIN"].RPN_POST_NMS_TOP_N, 4), dynamic_axes=[Axis.default_batch_axis()])
rpn_rois_buf = alias(rpn_rois_input, name='rpn_rois')
# base image classification model (e.g. VGG16 or AlexNet)
base_model = load_model(cfg['BASE_MODEL_PATH'])
print("stage 1a - rpn")
if True:
# Create initial network, only rpn, without detection network
# initial weights train?
# conv: base_model only conv3_1 and up
# rpn: init new yes
# frcn: - -
# conv layers
conv_layers = clone_conv_layers(base_model, cfg)
conv_out = conv_layers(feat_norm)
# RPN and losses
rpn_rois, rpn_losses = create_rpn(conv_out, scaled_gt_boxes, dims_node, cfg)
stage1_rpn_network = combine([rpn_rois, rpn_losses])
# train
if debug_output: plot(stage1_rpn_network, os.path.join(cfg.OUTPUT_PATH, "graph_frcn_train_stage1a_rpn." + cfg["CNTK"].GRAPH_TYPE))
train_model(image_input, roi_input, dims_input, rpn_losses, rpn_losses,
rpn_lr_per_sample_scaled, mm_schedule, l2_reg_weight, rpn_epochs, cfg)
print("stage 1a - buffering rpn proposals")
buffered_proposals_s1 = compute_rpn_proposals(stage1_rpn_network, image_input, roi_input, dims_input, cfg)
print("stage 1b - frcn")
if True:
# Create full network, initialize conv layers with imagenet, fix rpn weights
# initial weights train?
# conv: base_model only conv3_1 and up
# rpn: stage1a rpn model no --> use buffered proposals
# frcn: base_model + new yes
# conv_layers
conv_layers = clone_conv_layers(base_model, cfg)
conv_out = conv_layers(feat_norm)
# use buffered proposals in target layer
rois, label_targets, bbox_targets, bbox_inside_weights = \
create_proposal_target_layer(rpn_rois_buf, scaled_gt_boxes, cfg)
# Fast RCNN and losses
fc_layers = clone_model(base_model, [cfg["MODEL"].POOL_NODE_NAME], [cfg["MODEL"].LAST_HIDDEN_NODE_NAME], CloneMethod.clone)
cls_score, bbox_pred = create_fast_rcnn_predictor(conv_out, rois, fc_layers, cfg)
detection_losses = create_detection_losses(cls_score, label_targets, bbox_pred, rois, bbox_targets, bbox_inside_weights, cfg)
pred_error = classification_error(cls_score, label_targets, axis=1, name="pred_error")
stage1_frcn_network = combine([rois, cls_score, bbox_pred, detection_losses, pred_error])
# train
if debug_output: plot(stage1_frcn_network, os.path.join(cfg.OUTPUT_PATH, "graph_frcn_train_stage1b_frcn." + cfg["CNTK"].GRAPH_TYPE))
train_model(image_input, roi_input, dims_input, detection_losses, pred_error,
frcn_lr_per_sample_scaled, mm_schedule, l2_reg_weight, frcn_epochs, cfg,
rpn_rois_input=rpn_rois_input, buffered_rpn_proposals=buffered_proposals_s1)
buffered_proposals_s1 = None
print("stage 2a - rpn")
if True:
# Keep conv weights from detection network and fix them
# initial weights train?
# conv: stage1b frcn model no
# rpn: stage1a rpn model yes
# frcn: - -
# conv_layers
conv_layers = clone_model(stage1_frcn_network, [feature_node_name], [last_conv_node_name], CloneMethod.freeze)
conv_out = conv_layers(image_input)
# RPN and losses
rpn = clone_model(stage1_rpn_network, [last_conv_node_name, "roi_input", "dims_input"], ["rpn_rois", "rpn_losses"], CloneMethod.clone)
rpn_net = rpn(conv_out, dims_node, scaled_gt_boxes)
rpn_rois = rpn_net.outputs[0]
rpn_losses = rpn_net.outputs[1]
stage2_rpn_network = combine([rpn_rois, rpn_losses])
# train
if debug_output: plot(stage2_rpn_network, os.path.join(cfg.OUTPUT_PATH, "graph_frcn_train_stage2a_rpn." + cfg["CNTK"].GRAPH_TYPE))
train_model(image_input, roi_input, dims_input, rpn_losses, rpn_losses,
rpn_lr_per_sample_scaled, mm_schedule, l2_reg_weight, rpn_epochs, cfg)
print("stage 2a - buffering rpn proposals")
buffered_proposals_s2 = compute_rpn_proposals(stage2_rpn_network, image_input, roi_input, dims_input, cfg)
print("stage 2b - frcn")
if True:
# Keep conv and rpn weights from step 3 and fix them
# initial weights train?
# conv: stage2a rpn model no
# rpn: stage2a rpn model no --> use buffered proposals
# frcn: stage1b frcn model yes -
# conv_layers
conv_layers = clone_model(stage2_rpn_network, [feature_node_name], [last_conv_node_name], CloneMethod.freeze)
conv_out = conv_layers(image_input)
# Fast RCNN and losses
frcn = clone_model(stage1_frcn_network, [last_conv_node_name, "rpn_rois", "roi_input"],
["cls_score", "bbox_regr", "rpn_target_rois", "detection_losses", "pred_error"], CloneMethod.clone)
stage2_frcn_network = frcn(conv_out, rpn_rois_buf, scaled_gt_boxes)
detection_losses = stage2_frcn_network.outputs[3]
pred_error = stage2_frcn_network.outputs[4]
# train
if debug_output: plot(stage2_frcn_network, os.path.join(cfg.OUTPUT_PATH, "graph_frcn_train_stage2b_frcn." + cfg["CNTK"].GRAPH_TYPE))
train_model(image_input, roi_input, dims_input, detection_losses, pred_error,
frcn_lr_per_sample_scaled, mm_schedule, l2_reg_weight, frcn_epochs, cfg,
rpn_rois_input=rpn_rois_input, buffered_rpn_proposals=buffered_proposals_s2)
buffered_proposals_s2 = None
# resetting config values to original test values
cfg["TEST"].RPN_PRE_NMS_TOP_N = test_pre
cfg["TEST"].RPN_POST_NMS_TOP_N = test_post
return create_faster_rcnn_eval_model(stage2_frcn_network, image_input, dims_input, cfg, rpn_model=stage2_rpn_network)
def lstm_w_attention(h, c, x):
# alias is used to work around bug when arguments in block funcion are the same
attended = mha(h, encoded, C.alias(encoded))
xx = C.splice(attended, x)
return lstm(h, c, xx)
def create_network(input_vocab_dim, label_vocab_dim):
# network complexity; initially low for faster testing
hidden_dim = 256
num_layers = 1
# Source and target inputs to the model
input_seq_axis = Axis('inputAxis')
label_seq_axis = Axis('labelAxis')
raw_input = sequence.input(shape=(input_vocab_dim), sequence_axis=input_seq_axis, name='raw_input')
raw_labels = sequence.input(shape=(label_vocab_dim), sequence_axis=label_seq_axis, name='raw_labels')
# Instantiate the sequence to sequence translation model
input_sequence = raw_input
# Drop the sentence start token from the label, for decoder training
label_sequence = sequence.slice(raw_labels, 1, 0) # <s> A B C </s> --> A B C </s>
label_sentence_start = sequence.first(raw_labels) # <s>
is_first_label = sequence.is_first(label_sequence) # <s> 0 0 0 ...
label_sentence_start_scattered = sequence.scatter(
label_sentence_start, is_first_label)
# Encoder
encoder_outputH = stabilize(input_sequence)
for i in range(0, num_layers):
(encoder_outputH, encoder_outputC) = LSTMP_component_with_self_stabilization(
encoder_outputH.output, hidden_dim, hidden_dim, future_value, future_value)
thought_vectorH = sequence.first(encoder_outputH)
thought_vectorC = sequence.first(encoder_outputC)
thought_vector_broadcastH = sequence.broadcast_as(
thought_vectorH, label_sequence)
thought_vector_broadcastC = sequence.broadcast_as(
thought_vectorC, label_sequence)
# Decoder
decoder_history_hook = alias(label_sequence, name='decoder_history_hook') # copy label_sequence
decoder_input = element_select(is_first_label, label_sentence_start_scattered, past_value(
decoder_history_hook))
decoder_outputH = stabilize(decoder_input)
for i in range(0, num_layers):
if (i > 0):
recurrence_hookH = past_value
recurrence_hookC = past_value
else:
isFirst = sequence.is_first(label_sequence)
recurrence_hookH = lambda operand: element_select(
isFirst, thought_vector_broadcastH, past_value(operand))
recurrence_hookC = lambda operand: element_select(
isFirst, thought_vector_broadcastC, past_value(operand))
(decoder_outputH, encoder_outputC) = LSTMP_component_with_self_stabilization(
decoder_outputH.output, hidden_dim, hidden_dim, recurrence_hookH, recurrence_hookC)
decoder_output = decoder_outputH
# Softmax output layer
z = linear_layer(stabilize(decoder_output), label_vocab_dim)
# Criterion nodes
ce = cross_entropy_with_softmax(z, label_sequence)
errs = classification_error(z, label_sequence)
# network output for decoder history
net_output = hardmax(z)
# make a clone of the graph where the ground truth is replaced by the network output
ng = z.clone(CloneMethod.share, {decoder_history_hook.output : net_output.output})
return {
'raw_input' : raw_input,
'raw_labels' : raw_labels,
'ce' : ce,
'pe' : errs,
'ng' : ng,
'output': z
}
def train_faster_rcnn_alternating(cfg):
'''
4-Step Alternating Training scheme from the Faster R-CNN paper:
# Create initial network, only rpn, without detection network
# --> train only the rpn (and conv3_1 and up for VGG16)
# buffer region proposals from rpn
# Create full network, initialize conv layers with imagenet, use buffered proposals
# --> train only detection network (and conv3_1 and up for VGG16)
# Keep conv weights from detection network and fix them
# --> train only rpn
# buffer region proposals from rpn
# Keep conv and rpn weights from step 3 and fix them
# --> train only detection network
'''
# setting pre- and post-nms top N to training values since buffered proposals are used for further training
test_pre = cfg["TEST"].RPN_PRE_NMS_TOP_N
test_post = cfg["TEST"].RPN_POST_NMS_TOP_N
cfg["TEST"].RPN_PRE_NMS_TOP_N = cfg["TRAIN"].RPN_PRE_NMS_TOP_N
cfg["TEST"].RPN_POST_NMS_TOP_N = cfg["TRAIN"].RPN_POST_NMS_TOP_N
# Learning parameters
rpn_lr_factor = cfg["MODEL"].RPN_LR_FACTOR
rpn_lr_per_sample_scaled = [x * rpn_lr_factor for x in cfg["CNTK"].RPN_LR_PER_SAMPLE]
frcn_lr_factor = cfg["MODEL"].FRCN_LR_FACTOR
frcn_lr_per_sample_scaled = [x * frcn_lr_factor for x in cfg["CNTK"].FRCN_LR_PER_SAMPLE]
l2_reg_weight = cfg["CNTK"].L2_REG_WEIGHT
mm_schedule = momentum_schedule(cfg["CNTK"].MOMENTUM_PER_MB)
rpn_epochs = cfg["CNTK"].RPN_EPOCHS
frcn_epochs = cfg["CNTK"].FRCN_EPOCHS
feature_node_name = cfg["MODEL"].FEATURE_NODE_NAME
last_conv_node_name = cfg["MODEL"].LAST_CONV_NODE_NAME
print("Using base model: {}".format(cfg["MODEL"].BASE_MODEL))
print("rpn_lr_per_sample: {}".format(rpn_lr_per_sample_scaled))
print("frcn_lr_per_sample: {}".format(frcn_lr_per_sample_scaled))
debug_output=cfg["CNTK"].DEBUG_OUTPUT
if debug_output:
print("Storing graphs and models to %s." % cfg.OUTPUT_PATH)
# Input variables denoting features, labeled ground truth rois (as 5-tuples per roi) and image dimensions
image_input = input_variable(shape=(cfg.NUM_CHANNELS, cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH),
dynamic_axes=[Axis.default_batch_axis()],
name=feature_node_name)
feat_norm = image_input - Constant([[[v]] for v in cfg["MODEL"].IMG_PAD_COLOR])
roi_input = input_variable((cfg.INPUT_ROIS_PER_IMAGE, 5), dynamic_axes=[Axis.default_batch_axis()])
scaled_gt_boxes = alias(roi_input, name='roi_input')
dims_input = input_variable((6), dynamic_axes=[Axis.default_batch_axis()])
dims_node = alias(dims_input, name='dims_input')
rpn_rois_input = input_variable((cfg["TRAIN"].RPN_POST_NMS_TOP_N, 4), dynamic_axes=[Axis.default_batch_axis()])
rpn_rois_buf = alias(rpn_rois_input, name='rpn_rois')
# base image classification model (e.g. VGG16 or AlexNet)
base_model = load_model(cfg['BASE_MODEL_PATH'])
print("stage 1a - rpn")
if True:
# Create initial network, only rpn, without detection network
# initial weights train?
# conv: base_model only conv3_1 and up
# rpn: init new yes
# frcn: - -
# conv layers
conv_layers = clone_conv_layers(base_model, cfg)
conv_out = conv_layers(feat_norm)
# RPN and losses
rpn_rois, rpn_losses = create_rpn(conv_out, scaled_gt_boxes, dims_node, cfg)
stage1_rpn_network = combine([rpn_rois, rpn_losses])
# train
if debug_output: plot(stage1_rpn_network, os.path.join(cfg.OUTPUT_PATH, "graph_frcn_train_stage1a_rpn." + cfg["CNTK"].GRAPH_TYPE))
train_model(image_input, roi_input, dims_input, rpn_losses, rpn_losses,
rpn_lr_per_sample_scaled, mm_schedule, l2_reg_weight, rpn_epochs, cfg)
print("stage 1a - buffering rpn proposals")
buffered_proposals_s1 = compute_rpn_proposals(stage1_rpn_network, image_input, roi_input, dims_input, cfg)
print("stage 1b - frcn")
if True:
# Create full network, initialize conv layers with imagenet, fix rpn weights
# initial weights train?
# conv: base_model only conv3_1 and up
# rpn: stage1a rpn model no --> use buffered proposals
# frcn: base_model + new yes
# conv_layers
conv_layers = clone_conv_layers(base_model, cfg)
conv_out = conv_layers(feat_norm)
# use buffered proposals in target layer
rois, label_targets, bbox_targets, bbox_inside_weights = \
create_proposal_target_layer(rpn_rois_buf, scaled_gt_boxes, cfg)
# Fast RCNN and losses
fc_layers = clone_model(base_model, [cfg["MODEL"].POOL_NODE_NAME], [cfg["MODEL"].LAST_HIDDEN_NODE_NAME], CloneMethod.clone)
cls_score, bbox_pred = create_fast_rcnn_predictor(conv_out, rois, fc_layers, cfg)
detection_losses = create_detection_losses(cls_score, label_targets, bbox_pred, rois, bbox_targets, bbox_inside_weights, cfg)
pred_error = classification_error(cls_score, label_targets, axis=1, name="pred_error")
stage1_frcn_network = combine([rois, cls_score, bbox_pred, detection_losses, pred_error])
# train
if debug_output: plot(stage1_frcn_network, os.path.join(cfg.OUTPUT_PATH, "graph_frcn_train_stage1b_frcn." + cfg["CNTK"].GRAPH_TYPE))
train_model(image_input, roi_input, dims_input, detection_losses, pred_error,
frcn_lr_per_sample_scaled, mm_schedule, l2_reg_weight, frcn_epochs, cfg,
rpn_rois_input=rpn_rois_input, buffered_rpn_proposals=buffered_proposals_s1)
buffered_proposals_s1 = None
print("stage 2a - rpn")
if True:
# Keep conv weights from detection network and fix them
# initial weights train?
# conv: stage1b frcn model no
# rpn: stage1a rpn model yes
# frcn: - -
# conv_layers
conv_layers = clone_model(stage1_frcn_network, [feature_node_name], [last_conv_node_name], CloneMethod.freeze)
conv_out = conv_layers(image_input)
# RPN and losses
rpn = clone_model(stage1_rpn_network, [last_conv_node_name, "roi_input", "dims_input"], ["rpn_rois", "rpn_losses"], CloneMethod.clone)
rpn_net = rpn(conv_out, dims_node, scaled_gt_boxes)
rpn_rois = rpn_net.outputs[0]
rpn_losses = rpn_net.outputs[1]
stage2_rpn_network = combine([rpn_rois, rpn_losses])
# train
if debug_output: plot(stage2_rpn_network, os.path.join(cfg.OUTPUT_PATH, "graph_frcn_train_stage2a_rpn." + cfg["CNTK"].GRAPH_TYPE))
train_model(image_input, roi_input, dims_input, rpn_losses, rpn_losses,
rpn_lr_per_sample_scaled, mm_schedule, l2_reg_weight, rpn_epochs, cfg)
print("stage 2a - buffering rpn proposals")
buffered_proposals_s2 = compute_rpn_proposals(stage2_rpn_network, image_input, roi_input, dims_input, cfg)
print("stage 2b - frcn")
if True:
# Keep conv and rpn weights from step 3 and fix them
# initial weights train?
# conv: stage2a rpn model no
# rpn: stage2a rpn model no --> use buffered proposals
# frcn: stage1b frcn model yes -
# conv_layers
conv_layers = clone_model(stage2_rpn_network, [feature_node_name], [last_conv_node_name], CloneMethod.freeze)
conv_out = conv_layers(image_input)
# Fast RCNN and losses
frcn = clone_model(stage1_frcn_network, [last_conv_node_name, "rpn_rois", "roi_input"],
["cls_score", "bbox_regr", "rpn_target_rois", "detection_losses", "pred_error"], CloneMethod.clone)
stage2_frcn_network = frcn(conv_out, rpn_rois_buf, scaled_gt_boxes)
detection_losses = stage2_frcn_network.outputs[3]
pred_error = stage2_frcn_network.outputs[4]
# train
if debug_output: plot(stage2_frcn_network, os.path.join(cfg.OUTPUT_PATH, "graph_frcn_train_stage2b_frcn." + cfg["CNTK"].GRAPH_TYPE))
train_model(image_input, roi_input, dims_input, detection_losses, pred_error,
frcn_lr_per_sample_scaled, mm_schedule, l2_reg_weight, frcn_epochs, cfg,
rpn_rois_input=rpn_rois_input, buffered_rpn_proposals=buffered_proposals_s2)
buffered_proposals_s2 = None
# resetting config values to original test values
cfg["TEST"].RPN_PRE_NMS_TOP_N = test_pre
cfg["TEST"].RPN_POST_NMS_TOP_N = test_post
return create_faster_rcnn_eval_model(stage2_frcn_network, image_input, dims_input, cfg, rpn_model=stage2_rpn_network)
def train_faster_rcnn_alternating(base_model_file_name, debug_output=False):
'''
4-Step Alternating Training scheme from the Faster R-CNN paper:
# Create initial network, only rpn, without detection network
# --> train only the rpn (and conv3_1 and up for VGG16)
# buffer region proposals from rpn
# Create full network, initialize conv layers with imagenet, use buffered proposals
# --> train only detection network (and conv3_1 and up for VGG16)
# Keep conv weights from detection network and fix them
# --> train only rpn
# buffer region proposals from rpn
# Keep conv and rpn weights from step 3 and fix them
# --> train only detection network
'''
# Learning parameters
rpn_lr_factor = globalvars['rpn_lr_factor']
rpn_lr_per_sample_scaled = [x * rpn_lr_factor for x in cfg["CNTK"].RPN_LR_PER_SAMPLE]
frcn_lr_factor = globalvars['frcn_lr_factor']
frcn_lr_per_sample_scaled = [x * frcn_lr_factor for x in cfg["CNTK"].FRCN_LR_PER_SAMPLE]
l2_reg_weight = cfg["CNTK"].L2_REG_WEIGHT
mm_schedule = momentum_schedule(globalvars['momentum_per_mb'])
rpn_epochs = globalvars['rpn_epochs']
frcn_epochs = globalvars['frcn_epochs']
print("Using base model: {}".format(cfg["CNTK"].BASE_MODEL))
print("rpn_lr_per_sample: {}".format(rpn_lr_per_sample_scaled))
print("frcn_lr_per_sample: {}".format(frcn_lr_per_sample_scaled))
if debug_output:
print("Storing graphs and models to %s." % globalvars['output_path'])
# Input variables denoting features, labeled ground truth rois (as 5-tuples per roi) and image dimensions
image_input = input_variable((num_channels, image_height, image_width), dynamic_axes=[Axis.default_batch_axis()],
name=feature_node_name)
feat_norm = image_input - normalization_const
roi_input = input_variable((cfg["CNTK"].INPUT_ROIS_PER_IMAGE, 5), dynamic_axes=[Axis.default_batch_axis()])
scaled_gt_boxes = alias(roi_input, name='roi_input')
dims_input = input_variable((6), dynamic_axes=[Axis.default_batch_axis()])
dims_node = alias(dims_input, name='dims_input')
rpn_rois_input = input_variable((cfg["TRAIN"].RPN_POST_NMS_TOP_N, 4), dynamic_axes=[Axis.default_batch_axis()])
rpn_rois_buf = alias(rpn_rois_input, name='rpn_rois')
# base image classification model (e.g. VGG16 or AlexNet)
base_model = load_model(base_model_file_name)
print("stage 1a - rpn")
if True:
# Create initial network, only rpn, without detection network
# initial weights train?
# conv: base_model only conv3_1 and up
# rpn: init new yes
# frcn: - -
# conv layers
conv_layers = clone_conv_layers(base_model)
conv_out = conv_layers(feat_norm)
# RPN and losses
rpn_rois, rpn_losses = create_rpn(conv_out, scaled_gt_boxes, dims_node, proposal_layer_param_string=cfg["CNTK"].PROPOSAL_LAYER_PARAMS)
stage1_rpn_network = combine([rpn_rois, rpn_losses])
# train
if debug_output: plot(stage1_rpn_network, os.path.join(globalvars['output_path'], "graph_frcn_train_stage1a_rpn." + cfg["CNTK"].GRAPH_TYPE))
train_model(image_input, roi_input, dims_input, rpn_losses, rpn_losses,
rpn_lr_per_sample_scaled, mm_schedule, l2_reg_weight, epochs_to_train=rpn_epochs)
print("stage 1a - buffering rpn proposals")
buffered_proposals_s1 = compute_rpn_proposals(stage1_rpn_network, image_input, roi_input, dims_input)
print("stage 1b - frcn")
if True:
# Create full network, initialize conv layers with imagenet, fix rpn weights
# initial weights train?
# conv: base_model only conv3_1 and up
# rpn: stage1a rpn model no --> use buffered proposals
# frcn: base_model + new yes
# conv_layers
conv_layers = clone_conv_layers(base_model)
conv_out = conv_layers(feat_norm)
# use buffered proposals in target layer
rois, label_targets, bbox_targets, bbox_inside_weights = \
create_proposal_target_layer(rpn_rois_buf, scaled_gt_boxes, num_classes=globalvars['num_classes'])
# Fast RCNN and losses
fc_layers = clone_model(base_model, [pool_node_name], [last_hidden_node_name], CloneMethod.clone)
cls_score, bbox_pred = create_fast_rcnn_predictor(conv_out, rois, fc_layers)
detection_losses = create_detection_losses(cls_score, label_targets, rois, bbox_pred, bbox_targets, bbox_inside_weights)
pred_error = classification_error(cls_score, label_targets, axis=1, name="pred_error")
stage1_frcn_network = combine([rois, cls_score, bbox_pred, detection_losses, pred_error])
# train
if debug_output: plot(stage1_frcn_network, os.path.join(globalvars['output_path'], "graph_frcn_train_stage1b_frcn." + cfg["CNTK"].GRAPH_TYPE))
train_model(image_input, roi_input, dims_input, detection_losses, pred_error,
frcn_lr_per_sample_scaled, mm_schedule, l2_reg_weight, epochs_to_train=frcn_epochs,
rpn_rois_input=rpn_rois_input, buffered_rpn_proposals=buffered_proposals_s1)
buffered_proposals_s1 = None
print("stage 2a - rpn")
if True:
# Keep conv weights from detection network and fix them
# initial weights train?
# conv: stage1b frcn model no
# rpn: stage1a rpn model yes
# frcn: - -
# conv_layers
conv_layers = clone_model(stage1_frcn_network, [feature_node_name], [last_conv_node_name], CloneMethod.freeze)
conv_out = conv_layers(image_input)
# RPN and losses
rpn = clone_model(stage1_rpn_network, [last_conv_node_name, "roi_input", "dims_input"], ["rpn_rois", "rpn_losses"], CloneMethod.clone)
rpn_net = rpn(conv_out, dims_node, scaled_gt_boxes)
rpn_rois = rpn_net.outputs[0]
rpn_losses = rpn_net.outputs[1]
stage2_rpn_network = combine([rpn_rois, rpn_losses])
# train
if debug_output: plot(stage2_rpn_network, os.path.join(globalvars['output_path'], "graph_frcn_train_stage2a_rpn." + cfg["CNTK"].GRAPH_TYPE))
train_model(image_input, roi_input, dims_input, rpn_losses, rpn_losses,
rpn_lr_per_sample_scaled, mm_schedule, l2_reg_weight, epochs_to_train=rpn_epochs)
print("stage 2a - buffering rpn proposals")
buffered_proposals_s2 = compute_rpn_proposals(stage2_rpn_network, image_input, roi_input, dims_input)
print("stage 2b - frcn")
if True:
# Keep conv and rpn weights from step 3 and fix them
# initial weights train?
# conv: stage2a rpn model no
# rpn: stage2a rpn model no --> use buffered proposals
# frcn: stage1b frcn model yes -
# conv_layers
conv_layers = clone_model(stage2_rpn_network, [feature_node_name], [last_conv_node_name], CloneMethod.freeze)
conv_out = conv_layers(image_input)
# Fast RCNN and losses
frcn = clone_model(stage1_frcn_network, [last_conv_node_name, "rpn_rois", "roi_input"],
["cls_score", "bbox_regr", "rpn_target_rois", "detection_losses", "pred_error"], CloneMethod.clone)
stage2_frcn_network = frcn(conv_out, rpn_rois_buf, scaled_gt_boxes)
detection_losses = stage2_frcn_network.outputs[3]
pred_error = stage2_frcn_network.outputs[4]
# train
if debug_output: plot(stage2_frcn_network, os.path.join(globalvars['output_path'], "graph_frcn_train_stage2b_frcn." + cfg["CNTK"].GRAPH_TYPE))
train_model(image_input, roi_input, dims_input, detection_losses, pred_error,
frcn_lr_per_sample_scaled, mm_schedule, l2_reg_weight, epochs_to_train=frcn_epochs,
rpn_rois_input=rpn_rois_input, buffered_rpn_proposals=buffered_proposals_s2)
buffered_proposals_s2 = None
return create_eval_model(stage2_frcn_network, image_input, dims_input, rpn_model=stage2_rpn_network)
