def frcn_predictor(features, rois, n_classes, model_path):
# Load the pretrained classification net and find nodes
loaded_model = load_model(model_path)
feature_node = find_by_name(loaded_model, feature_node_name)
conv_node = find_by_name(loaded_model, last_conv_node_name)
pool_node = find_by_name(loaded_model, pool_node_name)
last_node = find_by_name(loaded_model, last_hidden_node_name)
# Clone the conv layers and the fully connected layers of the network
conv_layers = combine([conv_node.owner
]).clone(CloneMethod.freeze,
{feature_node: placeholder()})
fc_layers = combine([last_node.owner]).clone(CloneMethod.clone,
{pool_node: placeholder()})
# Create the Fast R-CNN model
feat_norm = features - Constant(114)
conv_out = conv_layers(feat_norm)
roi_out = roipooling(conv_out, rois, C.MAX_POOLING, (roi_dim, roi_dim),
0.0625)
fc_out = fc_layers(roi_out)
# z = Dense(rois[0], num_classes, map_rank=1)(fc_out) # --> map_rank=1 is not yet supported
W = parameter(shape=(4096, n_classes), init=glorot_uniform())
b = parameter(shape=n_classes, init=0)
z = times(fc_out, W) + b
return z
def create_detection_losses(cls_score, label_targets, rois, bbox_pred, bbox_targets, bbox_inside_weights):
# classification loss
cls_loss = cross_entropy_with_softmax(cls_score, label_targets, axis=1)
p_cls_loss = placeholder()
p_rois = placeholder()
# The terms that are accounted for in the cls loss are those that correspond to an actual roi proposal --> do not count no-op (all-zero) rois
roi_indicator = reduce_sum(p_rois, axis=1)
cls_num_terms = reduce_sum(cntk.greater_equal(roi_indicator, 0.0))
cls_normalization_factor = 1.0 / cls_num_terms
normalized_cls_loss = reduce_sum(p_cls_loss) * cls_normalization_factor
reduced_cls_loss = cntk.as_block(normalized_cls_loss,
[(p_cls_loss, cls_loss), (p_rois, rois)],
'Normalize', 'norm_cls_loss')
# regression loss
p_bbox_pred = placeholder()
p_bbox_targets = placeholder()
p_bbox_inside_weights = placeholder()
bbox_loss = SmoothL1Loss(cfg["CNTK"].SIGMA_DET_L1, p_bbox_pred, p_bbox_targets, p_bbox_inside_weights, 1.0)
# The bbox loss is normalized by the batch size
bbox_normalization_factor = 1.0 / cfg["TRAIN"].BATCH_SIZE
normalized_bbox_loss = reduce_sum(bbox_loss) * bbox_normalization_factor
reduced_bbox_loss = cntk.as_block(normalized_bbox_loss,
[(p_bbox_pred, bbox_pred), (p_bbox_targets, bbox_targets), (p_bbox_inside_weights, bbox_inside_weights)],
'SmoothL1Loss', 'norm_bbox_loss')
detection_losses = plus(reduced_cls_loss, reduced_bbox_loss, name="detection_losses")
return detection_losses
def create_detection_losses(cls_score, label_targets, bbox_pred, rois, bbox_targets, bbox_inside_weights, cfg):
# The losses are normalized by the batch size
# classification loss
p_cls_score = placeholder()
p_label_targets = placeholder()
cls_loss = cross_entropy_with_softmax(p_cls_score, p_label_targets, axis=1)
cls_normalization_factor = 1.0 / cfg.NUM_ROI_PROPOSALS
normalized_cls_loss = reduce_sum(cls_loss) * cls_normalization_factor
reduced_cls_loss = cntk.as_block(normalized_cls_loss,
[(p_cls_score, cls_score), (p_label_targets, label_targets)],
'CrossEntropyWithSoftmax', 'norm_cls_loss')
# regression loss
p_bbox_pred = placeholder()
p_bbox_targets = placeholder()
p_bbox_inside_weights = placeholder()
bbox_loss = SmoothL1Loss(cfg.SIGMA_DET_L1, p_bbox_pred, p_bbox_targets, p_bbox_inside_weights, 1.0)
bbox_normalization_factor = 1.0 / cfg.NUM_ROI_PROPOSALS
normalized_bbox_loss = reduce_sum(bbox_loss) * bbox_normalization_factor
reduced_bbox_loss = cntk.as_block(normalized_bbox_loss,
[(p_bbox_pred, bbox_pred), (p_bbox_targets, bbox_targets),
(p_bbox_inside_weights, bbox_inside_weights)],
'SmoothL1Loss', 'norm_bbox_loss')
detection_losses = plus(reduced_cls_loss, reduced_bbox_loss, name="detection_losses")
return detection_losses
def create_detection_losses(cls_score, label_targets, bbox_pred, rois, bbox_targets, bbox_inside_weights, cfg):
# The losses are normalized by the batch size
# classification loss
p_cls_score = placeholder()
p_label_targets = placeholder()
cls_loss = cross_entropy_with_softmax(p_cls_score, p_label_targets, axis=1)
cls_normalization_factor = 1.0 / cfg.NUM_ROI_PROPOSALS
normalized_cls_loss = reduce_sum(cls_loss) * cls_normalization_factor
reduced_cls_loss = cntk.as_block(normalized_cls_loss,
[(p_cls_score, cls_score), (p_label_targets, label_targets)],
'CrossEntropyWithSoftmax', 'norm_cls_loss')
# regression loss
p_bbox_pred = placeholder()
p_bbox_targets = placeholder()
p_bbox_inside_weights = placeholder()
bbox_loss = SmoothL1Loss(cfg.SIGMA_DET_L1, p_bbox_pred, p_bbox_targets, p_bbox_inside_weights, 1.0)
bbox_normalization_factor = 1.0 / cfg.NUM_ROI_PROPOSALS
normalized_bbox_loss = reduce_sum(bbox_loss) * bbox_normalization_factor
reduced_bbox_loss = cntk.as_block(normalized_bbox_loss,
[(p_bbox_pred, bbox_pred), (p_bbox_targets, bbox_targets), (p_bbox_inside_weights, bbox_inside_weights)],
'SmoothL1Loss', 'norm_bbox_loss')
detection_losses = plus(reduced_cls_loss, reduced_bbox_loss, name="detection_losses")
return detection_losses
def create_detection_losses(cls_score, label_targets, rois, bbox_pred, bbox_targets, bbox_inside_weights):
# classification loss
cls_loss = cross_entropy_with_softmax(cls_score, label_targets, axis=1)
p_cls_loss = placeholder()
p_rois = placeholder()
# The terms that are accounted for in the cls loss are those that correspond to an actual roi proposal --> do not count no-op (all-zero) rois
roi_indicator = reduce_sum(p_rois, axis=1)
cls_num_terms = reduce_sum(cntk.greater_equal(roi_indicator, 0.0))
cls_normalization_factor = 1.0 / cls_num_terms
normalized_cls_loss = reduce_sum(p_cls_loss) * cls_normalization_factor
reduced_cls_loss = cntk.as_block(normalized_cls_loss,
[(p_cls_loss, cls_loss), (p_rois, rois)],
'Normalize', 'norm_cls_loss')
# regression loss
p_bbox_pred = placeholder()
p_bbox_targets = placeholder()
p_bbox_inside_weights = placeholder()
bbox_loss = SmoothL1Loss(cfg["CNTK"].SIGMA_DET_L1, p_bbox_pred, p_bbox_targets, p_bbox_inside_weights, 1.0)
# The bbox loss is normalized by the batch size
bbox_normalization_factor = 1.0 / cfg["TRAIN"].BATCH_SIZE
normalized_bbox_loss = reduce_sum(bbox_loss) * bbox_normalization_factor
reduced_bbox_loss = cntk.as_block(normalized_bbox_loss,
[(p_bbox_pred, bbox_pred), (p_bbox_targets, bbox_targets), (p_bbox_inside_weights, bbox_inside_weights)],
'SmoothL1Loss', 'norm_bbox_loss')
detection_losses = plus(reduced_cls_loss, reduced_bbox_loss, name="detection_losses")
return detection_losses
def frcn_predictor(features, rois, n_classes, model_path):
# Load the pretrained classification net and find nodes
loaded_model = load_model(model_path)
feature_node = find_by_name(loaded_model, feature_node_name)
conv_node = find_by_name(loaded_model, last_conv_node_name)
pool_node = find_by_name(loaded_model, pool_node_name)
last_node = find_by_name(loaded_model, last_hidden_node_name)
# Clone the conv layers and the fully connected layers of the network
conv_layers = combine([conv_node.owner]).clone(CloneMethod.freeze, {feature_node: placeholder()})
fc_layers = combine([last_node.owner]).clone(CloneMethod.clone, {pool_node: placeholder()})
# Create the Fast R-CNN model
feat_norm = features - Constant(114)
conv_out = conv_layers(feat_norm)
roi_out = roipooling(conv_out, rois, (roi_dim, roi_dim))
fc_out = fc_layers(roi_out)
# z = Dense(rois[0], num_classes, map_rank=1)(fc_out) # --> map_rank=1 is not yet supported
W = parameter(shape=(4096, n_classes), init=glorot_uniform())
b = parameter(shape=n_classes, init=0)
z = times(fc_out, W) + b
return z
def frcn_predictor(features, rois, n_classes, base_path):
# model specific variables for AlexNet
model_file = base_path + "/../../../resources/cntk/AlexNet.model"
roi_dim = 6
feature_node_name = "features"
last_conv_node_name = "conv5.y"
pool_node_name = "pool3"
last_hidden_node_name = "h2_d"
# Load the pretrained classification net and find nodes
print("Loading pre-trained model...")
loaded_model = load_model(model_file)
print("Loading pre-trained model... DONE.")
feature_node = find_by_name(loaded_model, feature_node_name)
conv_node = find_by_name(loaded_model, last_conv_node_name)
pool_node = find_by_name(loaded_model, pool_node_name)
last_node = find_by_name(loaded_model, last_hidden_node_name)
# Clone the conv layers and the fully connected layers of the network
conv_layers = combine([conv_node.owner]).clone(CloneMethod.freeze, {feature_node: placeholder()})
fc_layers = combine([last_node.owner]).clone(CloneMethod.clone, {pool_node: placeholder()})
# Create the Fast R-CNN model
feat_norm = features - constant(114)
conv_out = conv_layers(feat_norm)
roi_out = roipooling(conv_out, rois, (roi_dim, roi_dim))
fc_out = fc_layers(roi_out)
#fc_out.set_name("fc_out")
# z = Dense(rois[0], num_classes, map_rank=1)(fc_out) # --> map_rank=1 is not yet supported
W = parameter(shape=(4096, n_classes), init=glorot_uniform())
b = parameter(shape=n_classes, init=0)
z = times(fc_out, W) + b
return z, fc_out
def create_model(base_model_file,
input_features,
num_classes,
dropout_rate=0.5,
freeze_weights=False):
# Load the pretrained classification net and find nodes
base_model = load_model(base_model_file)
feature_node = find_by_name(base_model, 'features')
beforePooling_node = find_by_name(base_model, "z.x.x.r")
#graph.plot(base_model, filename="base_model.pdf") # Write graph visualization
# Clone model until right before the pooling layer, ie. until including z.x.x.r
modelCloned = combine([beforePooling_node.owner]).clone(
CloneMethod.freeze if freeze_weights else CloneMethod.clone,
{feature_node: placeholder(name='features')})
# Center the input around zero and set model input.
# Do this early, to avoid CNTK bug with wrongly estimated layer shapes
feat_norm = input_features - constant(114)
model = modelCloned(feat_norm)
# Pool over all spatial dimensions and add dropout layer
avgPool = GlobalAveragePooling(name="poolingLayer")(model)
if dropout_rate > 0:
avgPoolDrop = Dropout(dropout_rate)(avgPool)
else:
avgPoolDrop = avgPool
# Add new dense layer for class prediction
finalModel = Dense(num_classes, activation=None,
name="prediction")(avgPoolDrop)
return finalModel
def clone_model(base_model, from_node_names, to_node_names, clone_method):
from_nodes = [find_by_name(base_model, node_name) for node_name in from_node_names]
if None in from_nodes:
print("Error: could not find all specified 'from_nodes' in clone. Looking for {}, found {}"
.format(from_node_names, from_nodes))
to_nodes = [find_by_name(base_model, node_name) for node_name in to_node_names]
if None in to_nodes:
print("Error: could not find all specified 'to_nodes' in clone. Looking for {}, found {}"
.format(to_node_names, to_nodes))
input_placeholders = dict(zip(from_nodes, [placeholder() for x in from_nodes]))
cloned_net = combine(to_nodes).clone(clone_method, input_placeholders)
return cloned_net
def clone_model(base_model, from_node_names, to_node_names, clone_method):
from_nodes = [find_by_name(base_model, node_name) for node_name in from_node_names]
if None in from_nodes:
print("Error: could not find all specified 'from_nodes' in clone. Looking for {}, found {}"
.format(from_node_names, from_nodes))
to_nodes = [find_by_name(base_model, node_name) for node_name in to_node_names]
if None in to_nodes:
print("Error: could not find all specified 'to_nodes' in clone. Looking for {}, found {}"
.format(to_node_names, to_nodes))
input_placeholders = dict(zip(from_nodes, [placeholder() for x in from_nodes]))
cloned_net = combine(to_nodes).clone(clone_method, input_placeholders)
return cloned_net
def create_tf_model(model_details,
num_classes,
input_features,
new_prediction_node_name='prediction',
freeze=False):
base_model = load_model(os.path.abspath(model_details['model_file']))
feature_node = logging.find_by_name(base_model,
model_details['feature_node_name'])
last_node = logging.find_by_name(base_model,
model_details['last_hidden_node_name'])
cloned_layers = combine([last_node.owner]).clone(
CloneMethod.freeze if freeze else CloneMethod.clone,
{feature_node: placeholder(name='features')})
cloned_out = cloned_layers(input_features)
z = Dense(num_classes, activation=None,
name=new_prediction_node_name)(cloned_out)
return z
def create_tf_model(model_details,
num_classes,
input_features,
new_prediction_node_name='prediction',
freeze=False):
# Load the pretrained classification net and find nodes
print(os.path.abspath(model_details['model_file']))
base_model = load_model(os.path.abspath(model_details['model_file']))
feature_node = logging.find_by_name(base_model,
model_details['feature_node_name'])
last_node = logging.find_by_name(base_model,
model_details['last_hidden_node_name'])
# Clone the desired layers with fixed weights
cloned_layers = combine([last_node.owner]).clone(
CloneMethod.freeze if freeze else CloneMethod.clone,
{feature_node: placeholder(name='features')})
# Add new dense layer for class prediction
cloned_out = cloned_layers(input_features)
z = Dense(num_classes, activation=None,
name=new_prediction_node_name)(cloned_out)
return z
