def modeltest(hdf5s, hdf5t, batch_size):
#logistic regression: data, matrix multiplication, and 2-class softmax loss
n = caffe.NetSpec()
#n.data, n.lp_label, n.bag_label = L.HDF5Data(batch_size=batch_size, source=hdf5t, ntop=3)
#n.data, n.lp_label, n.instance_label = L.HDF5Data(batch_size=batch_size, source=hdf5t, ntop=3)
#n.data, n.lp_label = L.HDF5Data(batch_size=batch_size, source=hdf5t, ntop=2)
n.data, n.lp_label, n.bag_label, n.instance_label = L.HDF5Data(batch_size=batch_size, source=hdf5t, ntop=4)
n.dc_label=L.DummyData(data_filler=dict(type='constant', value=1), num=batch_size, channels=1, height=1, width=1)
n.ip1 = L.InnerProduct(n.data, num_output=neuronL1, weight_filler=dict(type='xavier'))
n.relu1 = L.Sigmoid(n.ip1, in_place=True)
#n.dropout1 = L.Dropout(n.relu1, dropout_ratio=0.5)
n.ip2 = L.InnerProduct(n.relu1, num_output=neuronL1-400, weight_filler=dict(type='xavier'))
n.target_feature=L.Split(n.ip2)
n.ip4 = L.InnerProduct(n.target_feature, num_output=1, weight_filler=dict(type='xavier'))
#n.ip5=L.Sigmoid(n.ip4, in_place=True)
#n.real, n.ip3 = L.Python(n.source_feature, n.lp_label, n.bag_label, module= 'missSVM', layer='missSVMLayer', ntop=2)
#n.ip3 = L.InnerProduct(n.source_feature, num_output=1, weight_filler=dict(type='xavier'))
#n.accuracy = L.Accuracy(n.ip4, n.lp_label)
#L.Silence(n.bag_label);
#n.losslp = L.Python(n.ip4, n.lp_label, n.bag_label, module = 'GMloss', layer='MultipleInstanceLossLayer')
#n.P , n.Y = L.Python(n.ip4, n.lp_label, n.bag_label, module = 'MIloss', layer='MultipleInstanceLossLayer', ntop=2)
#n.losslp = L.SigmoidCrossEntropyLoss(n.P, n.Y)
n.losslp = L.SigmoidCrossEntropyLoss(n.ip4, n.lp_label)
#n.losstlp = L.SigmoidCrossEntropyLoss(n.ip4, n.lp_label)
n.grl= L.GradientScaler(n.ip2, lower_bound=0.0)
n.ip11 = L.InnerProduct(n.grl, num_output=300, weight_filler=dict(type='xavier'))
n.relu11 = L.Sigmoid(n.ip11, in_place=True)
n.dropout11 = L.Dropout(n.relu11, dropout_ratio=0.5)
n.ip12 = L.InnerProduct(n.dropout11, num_output=1, weight_filler=dict(type='xavier'))
n.lossdc = L.SigmoidCrossEntropyLoss(n.ip12, n.dc_label, loss_weight=0.1)
return n.to_proto()
def L_SigmoidCrossEntropyLoss(input_scores, input_labels, ignore_label=None):
if ignore_label is None:
output = L.SigmoidCrossEntropyLoss(input_scores, input_labels)
else:
output = L.SigmoidCrossEntropyLoss(
input_scores,
input_labels,
loss_param=dict(ignore_label=ignore_label))
return output
def caffenet_multilabel_sigmoid(name,
num_labels,
data_layer_params=None,
is_test=True):
# setup the ntb data layer
net = caffe.NetSpec()
assert is_test or data_layer_params
if is_test:
net.data = L.Input(input_param={"shape": {"dim": [1, 3, 227, 227]}})
else:
net.data, net.label = L.Python(module='ntb.layer.data',
layer='NTBDataLayer',
ntop=2,
param_str=str(data_layer_params))
net = add_caffenet(net, num_labels)
# Changed loss function
if is_test:
net.prob = L.Sigmoid(net.fc7)
else:
net.loss = L.SigmoidCrossEntropyLoss(net.score, net.label)
name_field = 'name: "{}"'.format(name)
return name_field + '\n' + str(net.to_proto())
def caffenet_multilabel_lmdb(name,
data_lmdb,
labels_lmdb,
num_labels,
mean_file,
batch_size=256,
mirror=False,
is_test=False):
assert is_test or data_lmdb and labels_lmdb and mean_file
net = caffe.NetSpec()
if is_test:
net.data = L.Input(input_param={"shape": {"dim": [1, 3, 227, 227]}})
else:
net.data = L.Data(source=data_lmdb,
backend=P.Data.LMDB,
batch_size=batch_size,
transform_param=dict(mean_file=mean_file,
mirror=mirror))
net.label = L.Data(
source=labels_lmdb,
backend=P.Data.LMDB,
batch_size=batch_size,
)
# the net itself
net = add_caffenet(net, num_labels)
# Changed loss function
if is_test:
net.prob = L.Sigmoid(net.fc7)
else:
net.loss = L.SigmoidCrossEntropyLoss(net.score, net.label)
name_field = 'name: "{}"'.format(name)
return name_field + '\n' + str(net.to_proto())
def add_sigmoid_entropy_loss(net, bottom, name, loss_weight, phase):
""" Add sigmoid entropy loss """
include_dict = {'phase': phase}
net[name] = L.SigmoidCrossEntropyLoss(bottom[0],
bottom[1],
loss_weight=loss_weight,
include=include_dict)
def caffenet_multilabel(data_layer_params, datalayer):
# setup the python data layer
n = caffe.NetSpec()
n.data, n.label = L.Python(module='pascal_multilabel_datalayers',
layer=datalayer,
ntop=2,
param_str=str(data_layer_params))
# the net itself
n.conv1, n.relu1 = conv_relu(n.data, 11, 96, stride=4)
n.pool1 = max_pool(n.relu1, 3, stride=2)
n.norm1 = L.LRN(n.pool1, local_size=5, alpha=1e-4, beta=0.75)
n.conv2, n.relu2 = conv_relu(n.norm1, 5, 256, pad=2, group=2)
n.pool2 = max_pool(n.relu2, 3, stride=2)
n.norm2 = L.LRN(n.pool2, local_size=5, alpha=1e-4, beta=0.75)
n.conv3, n.relu3 = conv_relu(n.norm2, 3, 384, pad=1)
n.conv4, n.relu4 = conv_relu(n.relu3, 3, 384, pad=1, group=2)
n.conv5, n.relu5 = conv_relu(n.relu4, 3, 256, pad=1, group=2)
n.pool5 = max_pool(n.relu5, 3, stride=2)
n.fc6, n.relu6 = fc_relu(n.pool5, 4096)
n.drop6 = L.Dropout(n.relu6, in_place=True)
n.fc7, n.relu7 = fc_relu(n.drop6, 4096)
n.drop7 = L.Dropout(n.relu7, in_place=True)
n.score = L.InnerProduct(n.drop7, num_output=20)
n.loss = L.SigmoidCrossEntropyLoss(n.score, n.label)
return str(n.to_proto())
def get_phocnet(self, word_image_lmdb_path, phoc_lmdb_path,
phoc_size=604, generate_deploy=False):
'''
Returns a NetSpec definition of the PHOCNet. The definition can then be transformed
into a protobuffer message by casting it into a str.
'''
n = NetSpec()
# Data
self.set_phocnet_data(n=n, generate_deploy=generate_deploy,
word_image_lmdb_path=word_image_lmdb_path,
phoc_lmdb_path=phoc_lmdb_path)
# Conv Part
self.set_phocnet_conv_body(n=n, relu_in_place=True)
# FC Part
n.spp5 = L.SPP(n.relu4_3, spp_param=dict(pool=P.SPP.MAX, pyramid_height=3, engine=self.spp_engine))
n.fc6, n.relu6, n.drop6 = self.fc_relu(bottom=n.spp5, layer_size=4096,
dropout_ratio=0.5, relu_in_place=True)
n.fc7, n.relu7, n.drop7 = self.fc_relu(bottom=n.drop6, layer_size=4096,
dropout_ratio=0.5, relu_in_place=True)
n.fc8 = L.InnerProduct(n.drop7, num_output=phoc_size,
weight_filler=dict(type=self.initialization),
bias_filler=dict(type='constant'))
n.sigmoid = L.Sigmoid(n.fc8, include=dict(phase=self.phase_test))
# output part
if not generate_deploy:
n.silence = L.Silence(n.sigmoid, ntop=0, include=dict(phase=self.phase_test))
n.loss = L.SigmoidCrossEntropyLoss(n.fc8, n.phocs)
return n.to_proto()
def stacked_hourglass_network(batch_size,
img_size,
nfeats,
multi,
out_dim,
include_acc=False):
data, label = L.MemoryData(batch_size=batch_size,
channels=3,
height=img_size,
width=img_size,
ntop=2,
include=dict(phase=0))
data = L.Input()
conv1 = conv_bn_relu(data, kernel_size=3, num_output=32, stride=2, pad=1)
r1 = residual_mobile(conv1, num_output=32, multi=2, num_input=32)
pool1 = L.Pooling(r1, pool=P.Pooling.MAX, stride=2, kernel_size=2)
r3 = residual_mobile(pool1, num_output=nfeats, multi=multi, num_input=32)
#
hg = hourglass_mobile(r3,
num_output=nfeats,
num_modual=4,
multi=multi,
num_input=nfeats)
hgr = residual_mobile(hg, num_output=nfeats, multi=multi, num_input=nfeats)
ll = conv_bn_relu(hgr, kernel_size=1, num_output=nfeats, stride=1, pad=0)
out = deconv(ll, num_output=out_dim, kernel_size=4, stride=2, pad=1)
loss = L.SigmoidCrossEntropyLoss(out, label)
if include_acc:
acc = L.Accuracy(out, label, include=dict(phase=1))
return to_proto(loss, acc)
else:
return to_proto(loss)
def multilabel_vgg_dictnet(data_layer_params, num_data):
n = caffe.NetSpec()
n.data, n.label = L.Python(module='multilabel_datalayers',
layer='MultilabelDataLayerSync',
ntop=2,
param_str=str(data_layer_params))
n.conv1, n.relu1 = conv_relu(n.data, 5, 64, pad=2)
n.pool1 = max_pool(n.relu1, 2, stride=2)
n.conv2, n.relu2 = conv_relu(n.pool1, 5, 128, pad=2)
n.pool2 = max_pool(n.relu2, 2, stride=2)
n.conv3, n.relu3 = conv_relu(n.pool2, 3, 256, pad=1)
n.conv3_5, n.relu3_5 = conv_relu(n.relu3, 3, 512, pad=1)
n.pool3 = max_pool(n.relu3_5, 2, stride=2)
n.conv4, n.relu4 = conv_relu(n.pool3, 3, 512, pad=1)
n.fc1, n.relu5 = fc_conv_relu(n.relu4, 13, 4, 4096)
n.drop6 = L.Dropout(n.relu5, in_place=True)
n.fc2, n.relu6 = fc_conv_relu(n.drop6, 1, 1, 4096)
n.drop7 = L.Dropout(n.relu6, in_place=True)
n.score = fc_conv(n.drop7, num_data)
n.loss = L.SigmoidCrossEntropyLoss(n.score, n.label)
return str(n.to_proto())
def conv1_autoencoder(split, batch_sz):
n = caffe.NetSpec()
n.data, n.label = L.ImageData(image_data_param=dict(source=split,
batch_size=batch_sz,
new_height=height,
new_width=width,
is_color=False),
ntop=2)
n.silence = L.Silence(n.label, ntop=0)
n.flatdata_i = L.Flatten(n.data)
n.conv1 = conv(n.data, 5, 5, 64, pad=2)
n.bn1 = L.BatchNorm(n.conv1,
use_global_stats=False,
in_place=True,
param=[{
"lr_mult": 0
}, {
"lr_mult": 0
}, {
"lr_mult": 0
}])
n.scale1 = L.Scale(n.bn1, bias_term=True, in_place=True)
n.relu1 = L.ReLU(n.scale1, relu_param=dict(negative_slope=0.1))
n.pool1 = max_pool(n.relu1, 2, stride=2)
n.code = conv(n.pool1, 5, 5, 64, pad=2)
n.upsample1 = L.Deconvolution(n.code,
param=dict(lr_mult=0, decay_mult=0),
convolution_param=dict(
group=64,
num_output=64,
kernel_size=4,
stride=2,
pad=1,
bias_term=False,
weight_filler=dict(type="bilinear")))
n.deconv1 = conv(n.upsample1, 5, 5, 1, pad=2)
n.debn1 = L.BatchNorm(n.deconv1,
use_global_stats=False,
in_place=True,
param=[{
"lr_mult": 0
}, {
"lr_mult": 0
}, {
"lr_mult": 0
}])
n.descale1 = L.Scale(n.debn1, bias_term=True, in_place=True)
n.derelu1 = L.ReLU(n.descale1, relu_param=dict(negative_slope=0.1))
n.flatdata_o = L.Flatten(n.derelu1)
n.loss_s = L.SigmoidCrossEntropyLoss(n.flatdata_o,
n.flatdata_i,
loss_weight=1)
n.loss_e = L.EuclideanLoss(n.flatdata_o, n.flatdata_i, loss_weight=0)
return str(n.to_proto())
def net(hdf5, batch_size):
n = caffe.NetSpec()
n.data, n.label = L.HDF5Data(batch_size=batch_size, source=hdf5, ntop=2)
n.ip1 = L.InnerProduct(n.data, num_output=50, weight_filler=dict(type='xavier'))
n.relu1 = L.ReLU(n.ip1, in_place=True)
n.ip2 = L.InnerProduct(n.relu1, num_output=4, weight_filler=dict(type='xavier'))
n.loss = L.SigmoidCrossEntropyLoss(n.ip2, n.label)
return n.to_proto()
def FCN(images_lmdb, labels_lmdb, batch_size, include_acc=False):
# net definition
n.data = L.Data(source=images_lmdb,
backend=P.Data.LMDB,
batch_size=batch_size,
ntop=1,
transform_param=dict(crop_size=0,
mean_value=[0],
mirror=False))
n.label = L.Data(source=labels_lmdb,
backend=P.Data.LMDB,
batch_size=batch_size,
ntop=1)
n.conv1, n.relu1 = conv_relu(n.data,
ks=5,
nout=100,
stride=1,
pad=0,
bias_value=0.1)
n.conv2, n.relu2 = conv_relu(n.conv1,
ks=3,
nout=200,
stride=1,
bias_value=0.1)
n.pool2 = max_pool(n.relu2, ks=2, stride=2)
n.conv3, n.relu3 = conv_relu(n.pool2,
ks=3,
nout=300,
stride=1,
bias_value=0.1)
n.conv4, n.relu4 = conv_relu(n.relu3,
ks=3,
nout=300,
stride=1,
bias_value=0.1)
n.pool4 = max_pool(n.relu4, ks=2, stride=2)
n.drop4 = L.Dropout(n.pool4, dropout_ratio=0.5, in_place=True)
n.fcc5 = L.InnerProduct(n.drop4, num_output=1000)
n.relu5 = L.ReLU(n.fcc5, in_place=True)
n.fcc6 = L.InnerProduct(n.fcc5, num_output=600)
# n.score_classes, _= conv_relu(n.drop4, ks=1, nout=2, weight_std=0.01, bias_value=0.1)
# n.upscore = L.Deconvolution(n.score_classes)
# n.score = L.Crop(n.upscore,n.data)
n.loss = L.SigmoidCrossEntropyLoss(n.fcc6,
n.label,
loss_param=dict(normalize=True))
# if include_acc:
# n.accuracy = L.Accuracy(n.score, n.label)
# return n.to_proto()
# else:
# return n.to_proto()
return n.to_proto()
def _semantic_regularization(self, xSemPr, xSemLb, semReg):
ns = self.netspec
if self.semantics == ATTRIBUTES:
name = 'SCoRe/semLoss'
ns[name] = L.SigmoidCrossEntropyLoss(
*[xSemPr, xSemLb],
name=name,
loss_weight=semReg / (len(self.constrains) * np.sqrt(2.)) *
10.,
include=dict(phase=caffe.TRAIN))
else:
c_keys = [key for key in self.constrains.keys()]
losses = ['SCoRe/semLoss/%s' % key for key in c_keys]
scores = ['SCoRe/semLoss/%s/scores' % key for key in c_keys]
labels = ['SCoRe/semLoss/%s/labels' % key for key in c_keys]
# Slice semantic scores
xSemPr_name = [k for k, v in ns.tops.iteritems() if v == xSemPr][0]
slice_scores = L.Slice(name='SCoRe/semLoss/slice_scores',
bottom=[xSemPr_name],
ntop=len(scores),
top=scores,
in_place=True,
slice_point=np.cumsum(
self.num_states)[:-1].tolist(),
include=dict(phase=caffe.TRAIN))
# Slice semantic labels
xSemLb_name = [k for k, v in ns.tops.iteritems() if v == xSemLb][0]
slice_labels = L.Slice(name='SCoRe/semLoss/slice_labels',
bottom=[xSemLb_name],
ntop=len(labels),
top=labels,
in_place=True,
slice_point=range(1, len(self.constrains)),
include=dict(phase=caffe.TRAIN))
# Add supervision to each slice
for i, xLoss in enumerate(losses):
ns[xLoss] = L.SoftmaxWithLoss(
*[slice_scores[i], slice_labels[i]],
name=xLoss,
loss_weight=semReg / len(self.constrains),
include=dict(phase=caffe.TRAIN))
# Summarize supervisions for display
ns['SCoRe/semLoss'] = L.Eltwise(
*[ns[l] for l in losses],
name='SCoRe/semLoss',
operation=P.Eltwise.SUM,
coeff=[semReg / len(self.constrains)] * len(losses),
include=dict(phase=caffe.TRAIN))
def caffenet(train_file, test_lmdb, input_dim, batch_size=20):
# Size of flattened array of single image
feats = np.prod(input_dim)
n = caffe.NetSpec()
# Define data layers
n.data, n.labels = L.ImageData(
batch_size=batch_size,
source=train_file,
# phase == 'TRAIN'
# include=[dict(phase=0)],
transform_param=dict(scale=1),
ntop=2)
# Unused test layer
# n.data_test = L.Data(name="data", batch_size=batch_size, backend=P.Data.LMDB, source=test_lmdb,
# # phase == 'TEST'
# include=[dict(phase=1)],
# transform_param=dict(scale=1./255), ntop=1)
n.flatdata = L.Flatten(n.data)
# Stack of Innerproduct->sigmoid layers
n.enc1 = encoder_layer(n.data, 1000)
n.encn1 = L.Sigmoid(n.enc1)
n.enc2 = encoder_layer(n.encn1, 500)
n.encn2 = L.Sigmoid(n.enc2)
n.enc3 = encoder_layer(n.encn2, 250)
n.encn3 = L.Sigmoid(n.enc3)
n.enc4 = encoder_layer(n.encn3, 30)
n.dec4 = encoder_layer(n.enc4, 250)
n.decn4 = L.Sigmoid(n.dec4)
n.dec3 = encoder_layer(n.decn4, 500)
n.decn3 = L.Sigmoid(n.dec3)
n.dec2 = encoder_layer(n.decn3, 1000)
n.decn2 = L.Sigmoid(n.dec2)
n.dec1 = encoder_layer(n.decn2, feats)
n.decn1 = L.Sigmoid(n.dec1)
n.sig_flat_data = L.Sigmoid(n.flatdata)
# Flatten the data so it can be compared to the output of the stack
# Loss layers
n.cross_entropy_loss = L.SigmoidCrossEntropyLoss(n.decn1, n.sig_flat_data)
n.euclidean_loss = L.EuclideanLoss(n.flatdata, n.decn1)
# n.f_out = L.Split(n.flatdata)
# Out layer
# n.out_layer = L.Split(n.data)
return n.to_proto()
def create_net(img_list, batch_size, include_acc=True):
# data,label=L.ImageData(source=img_list,batch_size=batch_size,new_width=48,new_height=48,ntop=2,
# transform_param=dict(crop_size=40,mirror=True))
# data,label=L.MemoryData(batch_size=batch_size,channels=3,height=256,width=256,ntop=2,
# include=dict(phase=0))
conv1 = L.Convolution(data,
kernel_size=5,
stride=1,
num_output=16,
pad=2,
weight_filler=dict(type='xavier'))
relu1 = L.ReLU(conv1, in_place=True)
pool1 = L.Pooling(relu1, pool=P.Pooling.MAX, kernel_size=3, stride=2)
conv2 = L.Convolution(pool1,
kernel_size=53,
stride=1,
num_output=32,
pad=1,
weight_filler=dict(type='xavier'))
relu2 = L.ReLU(conv2, in_place=True)
pool2 = L.Pooling(relu2, pool=P.Pooling.MAX, kernel_size=3, stride=2)
conv3 = L.Convolution(pool2,
kernel_size=53,
stride=1,
num_output=32,
pad=1,
weight_filler=dict(type='xavier'))
relu3 = L.ReLU(conv3, in_place=True)
pool3 = L.Pooling(relu3, pool=P.Pooling.MAX, kernel_size=3, stride=2)
fc4 = L.InnerProduct(pool3,
num_output=1024,
weight_filler=dict(type='xavier'))
relu4 = L.ReLU(fc4, in_place=True)
drop4 = L.Dropout(relu4, in_place=True)
fc5 = L.InnerProduct(drop4,
num_output=7,
weight_filler=dict(type='xavier'))
# loss = L.SoftmaxWithLoss(fc5, label)
loss = L.SigmoidCrossEntropyLoss(fc5, label)
if include_acc:
acc = L.Accuracy(fc5,
label,
top_k=5,
name='loss',
include=dict(phase=1))
return to_proto(loss, acc)
else:
return to_proto(loss)
def compile_time_operation(self, learning_option, cluster):
"""
define sigmoid cross entropy loss operation for input(logits) tensors
ourputs:
output: loss output
"""
# get input
logits = self.get_input('logits')
labels = self.get_input('labels')
loss = L.SigmoidCrossEntropyLoss(logits, labels, name=self.name)
# set output
self.set_output('output', loss)
def generate_scores(split, config):
n = caffe.NetSpec()
batch_size = config.N
mode_str = str(dict(split=split, batch_size=batch_size))
n.language, n.cont, n.img_feature, n.spatial, n.label = L.Python(module=config.data_provider,
layer='TossLayer',
param_str=mode_str,
ntop=5)
# embedding
n.embed = L.Embed(n.language, input_dim=config.vocab_size,
num_output=config.embed_dim,
weight_filler=dict(type='uniform', min=-0.08, max=0.08))
# LSTM
n.lstm = L.LSTM(n.embed, n.cont,
recurrent_param=dict(num_output=config.lstm_dim,
weight_filler=dict(type='uniform', min=-0.08, max=0.08),
bias_filler=dict(type='constant', value=0)))
tops = L.Slice(n.lstm, ntop=config.T, slice_param=dict(axis=0))
for i in range(config.T - 1):
n.__setattr__('slice'+str(i), tops[i])
n.__setattr__('silence'+str(i), L.Silence(tops[i], ntop=0))
n.lstm_out = tops[-1]
n.lstm_feat = L.Reshape(n.lstm_out, reshape_param=dict(shape=dict(dim=[-1, config.lstm_dim])))
# L2 Normalize image and language features
n.img_l2norm = L.L2Normalize(n.img_feature)
n.lstm_l2norm = L.L2Normalize(n.lstm_feat)
n.img_l2norm_resh = L.Reshape(n.img_l2norm,
reshape_param=dict(shape=dict(dim=[-1, config.D_im])))
n.lstm_l2norm_resh = L.Reshape(n.lstm_l2norm,
reshape_param=dict(shape=dict(dim=[-1, config.D_text])))
# Concatenate
n.feat_all = L.Concat(n.lstm_l2norm_resh, n.img_l2norm_resh, n.spatial, concat_param=dict(axis=1))
# MLP Classifier over concatenated feature
n.mlp_l1, n.mlp_relu1 = fc_relu(n.feat_all, config.mlp_hidden_dims)
if config.mlp_dropout:
n.mlp_drop1 = L.Dropout(n.mlp_relu1, dropout_ratio=0.5, in_place=True)
n.scores = fc(n.mlp_drop1, 1)
else:
n.scores = fc(n.mlp_relu1, 1)
# Loss Layer
n.loss = L.SigmoidCrossEntropyLoss(n.scores, n.label)
return n.to_proto()
def conv_pool_net():
n = caffe.NetSpec()
n.data = L.DummyData(dummy_data_param=dict(num=20,
channels=1,
height=64,
width=64,
data_filler=dict(
type="gaussian")))
n.label = L.DummyData(dummy_data_param=dict(num=20,
channels=10,
height=1,
width=1,
data_filler=dict(
type='gaussian')))
n.conv1 = L.Convolution(n.data,
num_output=20,
kernel_size=4,
stride=3,
pad=0) # (64-4)/3+1=21
n.relu1 = L.ReLU(n.conv1, in_place=True) # 21x21
n.pool1 = L.Pooling(n.relu1, pool=P.Pooling.MAX, kernel_size=2,
stride=2) # ceil((21-2)/2+1)=11
# iter conv net
for i in range(2):
# conv_name = 'conv' + str(i+2)
# relu_name = 'relu' + str(i+2)
# pool_name = 'pool' + str(i+2)
n.conv1 = L.Convolution(n.pool1,
num_output=10,
kernel_size=4,
stride=2,
pad=3) # int((11+3*2-4)/2+1)=7
n.relu1 = L.ReLU(n.conv1, in_place=True)
n.pool1 = L.Pooling(n.relu1,
pool=P.Pooling.MAX,
kernel_size=2,
stride=2)
n.ip2 = L.InnerProduct(n.pool1,
num_output=10,
weight_filler=dict(type='xavier'))
n.loss = L.SigmoidCrossEntropyLoss(n.ip2,
n.label) # SigmoidCrossEntropyLoss
return n.to_proto()
def net(datafile, labelfile, batch_size, mean_value=0):
n = caffe.NetSpec()
n.data = L.Data(source=datafile,
backend=P.Data.LMDB,
batch_size=batch_size,
ntop=1,
transform_param=dict(scale=1.0 / 30.0,
mean_value=mean_value))
n.label = L.Data(source=labelfile,
backend=P.Data.LMDB,
batch_size=batch_size,
ntop=1)
n.ip1 = L.InnerProduct(n.data,
num_output=50,
weight_filler=dict(type='xavier'))
n.relu1 = L.ReLU(n.ip1, in_place=True)
n.ip2 = L.InnerProduct(n.relu1,
num_output=10,
weight_filler=dict(type='xavier'))
n.loss = L.SigmoidCrossEntropyLoss(n.ip2, n.label)
return n.to_proto()
def caffenet_multilabel(data_layer_params, datalayer=None):
# setup the python data layer
n = caffe.NetSpec()
if data_layer_params['split'] != 'test':
n.data, n.label = L.Python(module='multilabel_datalayers',
layer=datalayer,
ntop=2,
param_str=str(data_layer_params))
else:
n.data = L.Input(shape=dict(dim=[1, 26, 31, 23]))
# the net itself
n.conv1, n.relu1 = conv_relu(n.data,
2,
96,
stride=1,
weight_filler={'type': 'xavier'})
n.pool1 = max_pool(n.relu1, 2, stride=2)
n.norm1 = L.LRN(n.pool1, local_size=5, alpha=1e-4, beta=0.75)
n.conv2, n.relu2 = conv_relu(n.norm1,
2,
128,
group=2,
weight_filler={'type': 'xavier'})
n.pool2 = max_pool(n.relu2, 2, stride=2)
n.norm2 = L.LRN(n.pool2, local_size=5, alpha=1e-4, beta=0.75)
# n.conv3, n.relu3 = conv_relu(n.norm2, 3, 384, pad=1)
# n.pool3 = max_pool(n.relu3, 3, stride=2)
# n.conv4, n.relu4 = conv_relu(n.relu3, 3, 384, pad=1, group=2)
# n.conv5, n.relu5 = conv_relu(n.relu4, 3, 256, pad=1, group=2)
# n.pool5 = max_pool(n.relu5, 3, stride=2)
n.fc6, n.relu6 = fc_relu(n.norm2, 300, weight_filler={'type': 'xavier'})
n.drop6 = L.Dropout(n.relu6, in_place=True)
n.fc7, n.relu7 = fc_relu(n.drop6, 300, weight_filler={'type': 'xavier'})
n.drop7 = L.Dropout(n.relu7, in_place=True)
n.score = L.InnerProduct(n.drop7, num_output=19)
if data_layer_params['split'] != 'test':
n.loss = L.SigmoidCrossEntropyLoss(n.score, n.label)
return str(n.to_proto())
def multilabel_vgg16(data_layer_params, num_data):
# setup the python data layer
n = caffe.NetSpec()
n.data, n.label = L.Python(module='multilabel_datalayers',
layer='MultilabelDataLayerSync',
ntop=2,
param_str=str(data_layer_params))
n.conv1_1, n.relu1_1 = conv_relu(n.data, 3, 64, pad=1)
n.conv1_2, n.relu1_2 = conv_relu(n.relu1_1, 3, 64)
n.pool1 = max_pool(n.relu1_2, 2, stride=2)
n.conv2_1, n.relu2_1 = conv_relu(n.pool1, 3, 128, pad=1)
n.conv2_2, n.relu2_2 = conv_relu(n.relu2_1, 3, 128, pad=1)
n.pool2 = max_pool(n.relu2_2, 2, stride=2)
n.conv3_1, n.relu3_1 = conv_relu(n.pool2, 3, 256, pad=1)
n.conv3_2, n.relu3_2 = conv_relu(n.relu3_1, 3, 256, pad=1)
n.conv3_3, n.relu3_3 = conv_relu(n.relu3_2, 3, 256, pad=1)
n.pool3 = max_pool(n.relu3_3, 2, stride=2)
n.conv4_1, n.relu4_1 = conv_relu(n.pool3, 3, 512, pad=1)
n.conv4_2, n.relu4_2 = conv_relu(n.relu4_1, 3, 512, pad=1)
n.conv4_3, n.relu4_3 = conv_relu(n.relu4_2, 3, 512, pad=1)
n.pool4 = max_pool(n.relu4_3, 2, stride=2)
n.conv5_1, n.relu5_1 = conv_relu(n.pool4, 3, 512, pad=1)
n.conv5_2, n.relu5_2 = conv_relu(n.relu5_1, 3, 512, pad=1)
n.conv5_3, n.relu5_3 = conv_relu(n.relu5_2, 3, 512, pad=1)
n.pool5 = max_pool(n.relu5_3, 2, stride=2)
n.fc6, n.relu6 = in_relu(n.pool5, 4096)
n.drop6 = L.Dropout(n.relu6, dropout_ratio=0.5, in_place=True)
n.fc7, n.relu7 = in_relu(n.drop6, 4096)
n.drop7 = L.Dropout(n.relu7, dropout_ratio=0.5, in_place=True)
n.out = L.InnerProduct(n.drop7, num_output=num_data)
n.loss = L.SigmoidCrossEntropyLoss(n.out, n.label)
return str(n.to_proto())
def conv_pool_net():
n = caffe.NetSpec()
n.data = L.DummyData(dummy_data_param=dict(num=20,
channels=1,
height=64,
width=64,
data_filler=dict(
type="gaussian")))
n.label = L.DummyData(dummy_data_param=dict(num=20,
channels=10,
height=1,
width=1,
data_filler=dict(
type="gaussian")))
n.conv1 = L.Convolution(n.data,
num_output=20,
kernel_size=4,
stride=3,
pad=0)
n.relu1 = L.ReLU(n.conv1, in_place=True)
n.pool1 = L.Pooling(n.relu1, pool=P.Pooling.MAX, kernel_size=2, stride=2)
# 当变量名相同时,caffe会自动将之前的变量都按自定义的方式命名,只有最后一次使用时才保留自己定义的名
for i in range(2):
n.conv1 = L.Convolution(n.pool1,
num_output=10,
kernel_size=4,
stride=2,
pad=3)
n.relu1 = L.ReLU(n.conv1, in_place=True)
n.pool1 = L.Pooling(n.relu1,
pool=P.Pooling.MAX,
kernel_size=2,
stride=2)
n.ip2 = L.InnerProduct(n.pool1,
num_output=10,
weight_filler=dict(type='xavier'))
n.loss = L.SigmoidCrossEntropyLoss(n.ip2, n.label)
return n.to_proto()
def generate_loss(kModel, n, label_name):
# Determine the loss needed to be added
for output in kModel.output_layers:
if hasattr(kModel, 'loss'):
if kModel.loss == 'categorical_crossentropy' and output.activation.__name__ == 'softmax':
name = output.name + "_activation_" + output.activation.__name__
n[name] = L.SoftmaxWithLoss(n[output.name], n[label_name])
elif kModel.loss == 'binary_crossentropy' and output.activation.__name__ == 'sigmoid':
name = output.name + "_activation_" + output.activation.__name__
n[name] = L.SigmoidCrossEntropyLoss(n[output.name])
else: # Map the rest of the loss functions to the end of the output layer in Keras
if kModel.loss == 'hinge':
name = kModel.name + 'hinge'
n[name] = L.HingeLoss(n[output.name])
elif kModel.loss == 'categorical_crossentropy':
name = kModel.name + 'categorical_crossentropy'
n[name] = L.MultinomialLogisticLoss(n[output.name])
# TODO Post warning to use softmax before this loss
elif kModel.loss == 'mean_squared_error':
name = kModel.name + 'mean_squared_error'
n[name] = L.EuclideanLoss(n[output.name])
# TODO implement Infogain Loss
else:
raise Exception(kModel.loss + "is not supported")
def LowABGAN(w, batchsize, n):
n.ABnothing = L.DummyData(shape=[dict(dim=[batchsize, 1, 1, 1])], data_filler=dict(type='constant'), ntop=1)
n.ABlabels = L.Python(n.ABnothing, python_param=dict(module='destroy', layer='DestroyLayer'))
codings = [8, 16, 24, 32, 40]
level = 2
listofsizes = []
if full_conv:
listofsizes = [w]
for i in range(0, level-1):
alast = listofsizes[i]
listofsizes.append((alast - 4)*2)
listofsizes[0] -= 4
d=w
outname = ""
for i in range(0,level):
if full_conv:
n["ABZrand_"+str(i)] = L.DummyData(shape=[dict(dim=[batchsize, 1, listofsizes[level-1 -i] + 4, listofsizes[level-1 -i] + 4])], data_filler=dict(type='uniform',min=0., max=255.), ntop=1)
else:
n["ABZrand_"+str(i)] = L.DummyData(shape=[dict(dim=[batchsize, 1, d, d])], data_filler=dict(type='uniform',min=0., max=255.), ntop=1)
n, outname = convBlock("ABconvA"+str(i), codings[0], n, "ABZrand_"+str(i), train=True)
d /= 2
n, outname = joinBlock("ABjoinA", codings[0], n, outname, 'gelu'+'ABconvA0'+'_3', train=True)
n, outname = convBlock("ABconvB", codings[1], n, outname, train=True)
convolution_param = dict(num_output=1, kernel_size=1, stride=1, pad=0, weight_filler = dict(type='xavier'))
n["ABtexture"] = L.Convolution(n[outname], convolution_param=convolution_param, name="Atextureparam")
#GAN network
convolution_param = dict(num_output=16, kernel_size=3, stride=1, pad=1, weight_filler = dict(type='xavier'))
n.ganABconv1 = L.Convolution(n["ABtexture"], param=[dict(lr_mult=0, decay_mult= 0),dict(lr_mult=0, decay_mult= 0)], convolution_param=convolution_param)
n.ganABconv1 = L.ReLU(n.ganABconv1, negative_slope=0.1)
convolution_param = dict(num_output=16, kernel_size=3, stride=1, pad=1, weight_filler = dict(type='xavier'))
n.ganABconv2 = L.Convolution(n.ganABconv1, param=[dict(lr_mult=0, decay_mult= 0),dict(lr_mult=0, decay_mult= 0)], convolution_param=convolution_param)
n.ganABconv2 = L.ReLU(n.ganABconv2, negative_slope=0.1)
convolution_param = dict(num_output=32, kernel_size=2, stride=2, pad=0, weight_filler = dict(type='xavier'))
n.ganABconv3 = L.Convolution(n.ganABconv2, param=[dict(lr_mult=0, decay_mult= 0),dict(lr_mult=0, decay_mult= 0)], convolution_param=convolution_param)
n.ganABconv3 = L.ReLU(n.ganABconv3, negative_slope=0.1)
convolution_param = dict(num_output=32, kernel_size=3, stride=1, pad=1, weight_filler = dict(type='xavier'))
n.ganABconv4 = L.Convolution(n.ganABconv3, param=[dict(lr_mult=0, decay_mult= 0),dict(lr_mult=0, decay_mult= 0)], convolution_param=convolution_param)
n.ganABconv4 = L.ReLU(n.ganABconv4, negative_slope=0.1)
convolution_param = dict(num_output=32, kernel_size=3, stride=1, pad=1, weight_filler = dict(type='xavier'))
n.ganABconv5 = L.Convolution(n.ganABconv4, param=[dict(lr_mult=0, decay_mult= 0),dict(lr_mult=0, decay_mult= 0)], convolution_param=convolution_param)
#n.ganAconv5 = L.BatchNorm(n.ganAconv5, use_global_stats=global_stat, name=allparamnames.pop(0))#, param=[{"lr_mult":0},{"lr_mult":0},{"lr_mult":0}])
n.ganABconv5 = L.ReLU(n.ganABconv5, negative_slope=0.1)
convolution_param = dict(num_output=32, kernel_size=2, stride=2, pad=0, weight_filler = dict(type='xavier'))
n.ganABconv6 = L.Convolution(n.ganABconv5, param=[dict(lr_mult=0, decay_mult= 0),dict(lr_mult=0, decay_mult= 0)], convolution_param=convolution_param)
#n.ganAconv6 = L.BatchNorm(n.ganAconv6, use_global_stats=global_stat, name=allparamnames.pop(0))#, param=[{"lr_mult":0},{"lr_mult":0},{"lr_mult":0}])
n.ganABconv6 = L.ReLU(n.ganABconv6, negative_slope=0.1)
convolution_param = dict(num_output=32, kernel_size=1, stride=1, pad=0, weight_filler = dict(type='xavier'))
n.ganABconv7 = L.Convolution(n.ganABconv6, param=[dict(lr_mult=0, decay_mult= 0),dict(lr_mult=0, decay_mult= 0)], convolution_param=convolution_param)
#n.ganAconv7 = L.BatchNorm(n.ganAconv7, use_global_stats=global_stat, name=allparamnames.pop(0))#, param=[{"lr_mult":0},{"lr_mult":0},{"lr_mult":0}])
n.ganABconv7 = L.ReLU(n.ganABconv7, negative_slope=0.1)
n.ganABconv7_pool = L.Pooling(n.ganABconv7, global_pooling=True, pool=P.Pooling.AVE)
n.ABip3 = L.InnerProduct(n.ganABconv7_pool, param=[dict(lr_mult=0, decay_mult= 0),dict(lr_mult=0, decay_mult= 0)], num_output=1, weight_filler=dict(type='xavier'), name="last")
n.ABloss = L.SigmoidCrossEntropyLoss(n.ABip3, n.ABlabels)
return n
def sigmoid_loss(self, bottom_data, bottom_label, loss_weight=1):
return L.SigmoidCrossEntropyLoss(bottom_data,
bottom_label,
loss_weight=loss_weight,
loss_param=dict(ignore_label=-1))
n.drpout1 = L.Dropout(n.relu1, dropout_ratio= 0.2)
n.conv2 = L.Convolution(n.drpout1, num_output=8, kernel_size= 5, pad=2, engine=1)
n.relu2 = L.ReLU(n.conv2, negative_slope=0.3, engine=1)
n.bn2 = L.BatchNorm(n.relu2, in_place=True)
n.drpout2 = L.Dropout(n.relu2, dropout_ratio= 0.2)
n.conv3 = L.Convolution(n.drpout2, num_output=8, kernel_size= 5, pad=2, engine=1)
n.relu3 = L.ReLU(n.conv3, negative_slope=0.3, engine=1)
n.bn3 = L.BatchNorm(n.relu3, in_place=True)
n.drpout3 = L.Dropout(n.bn3, dropout_ratio= 0.2)
n.conv4 = L.Convolution(n.drpout3, num_output=8, kernel_size= 5, pad=1, stride= 2, engine=1) # used stride instead of average pooling
n.relu4 = L.ReLU(n.conv4, negative_slope=0.3, engine=1)
n.bn4 = L.BatchNorm(n.relu4, in_place=True)
n.drpout4 = L.Dropout(n.bn4, dropout_ratio= 0.2)
#n.pool4 = L.Pooling(n.drpout4, kernel_size=2, stride=1, pool=P.Pooling.AVE)
n.tag = L.InnerProduct(n.drpout4, num_output = 1)
n.loss_tag = L.SigmoidCrossEntropyLoss(n.tag, n.TAG, loss_weight=0.5)
n.aux = L.InnerProduct(n.drpout4, num_output = 1)
n.loss_aux = L.SigmoidCrossEntropyLoss(n.aux, n.event, loss_weight=0.5)
with open('discriminator2.prototxt', 'w') as f:
f.write(str(n.to_proto()))
#############################################################################
#make solvers
with open ("solver_template2.prototxt", "r") as myfile:
solver_template=myfile.read()
for curr_net in sub_nets:
with open("solver_%s.prototxt" % curr_net, "w") as myfile:
def LowAGAN(w, batchsize, n):
#input w = 11 damit output 10 <= 8*10 ist maximum
#input w = 16 damit output 20 <= 4*20 ist maximum
level = 2
listofsizes = []
if full_conv:
listofsizes = [w]
for i in range(0, level-1):
alast = listofsizes[i]
listofsizes.append((alast - 4)*2)
listofsizes[0] -= 4
transform_param = dict(mirror=False, crop_size=w, scale=1., mean_value=103.939)
if full_conv:
transform_param = dict(mirror=False, crop_size=120, scale=1., mean_value=103.939)
n.Adata, n.Anothing = L.ImageData(transform_param=transform_param, source='datasource.txt',
is_color=False, shuffle=True, batch_size=batchsize, ntop=2)
n.Aresize = L.Python(n.Adata, python_param=dict(module='resizelayer', layer='ResizeData'), param_str=str(4))
n.Acropped = L.Python(n.Aresize, python_param=dict(module='randomrot', layer='RandomRotLayer'), param_str=str(listofsizes[level -1] - 4))
codings = [8, 16, 24, 32, 40]
d=w
outname = ""
for i in range(0,level):
if full_conv:
n["AZrand_"+str(i)] = L.DummyData(shape=[dict(dim=[batchsize, 1, listofsizes[level-1 -i] + 4, listofsizes[level-1 -i] + 4])], data_filler=dict(type='uniform',min=0., max=255.), ntop=1)
else:
n["AZrand_"+str(i)] = L.DummyData(shape=[dict(dim=[batchsize, 1, d, d])], data_filler=dict(type='uniform',min=0., max=255.), ntop=1)
n, outname = convBlock("AconvA"+str(i), codings[0], n, "AZrand_"+str(i), train=False)
d /= 2
n, outname = joinBlock("AjoinA", codings[0], n, outname, 'gelu'+'AconvA0'+'_3', train=False)
n, outname = convBlock("AconvB", codings[1], n, outname, train=False)
convolution_param = dict(num_output=1, kernel_size=1, stride=1, pad=0, weight_filler = dict(type='xavier'))
n["Atexture"] = L.Convolution(n[outname], convolution_param=convolution_param, param=[dict(lr_mult=0, decay_mult= 0),dict(lr_mult=0, decay_mult= 0)], name="Atextureparam")
#0 => blurdat
#1 => data
n.Aswap, n.Alabels = L.Python(n["Atexture"], n.Acropped, python_param=dict(module='swaplayer', layer='SwapLayer'), propagate_down=[False, False], ntop=2)
if full_conv:
n.Anoise = L.DummyData(shape=[dict(dim=[batchsize, 1, listofsizes[level -1] - 4, listofsizes[level -1] - 4])], data_filler=dict(type='gaussian', std=2.0), ntop=1)
else:
n.Anoise = L.DummyData(shape=[dict(dim=[batchsize, 1, w, w])], data_filler=dict(type='gaussian', std=2.0), ntop=1)
n.Ainp = L.Eltwise(n.Aswap, n.Anoise, eltwise_param={'operation':1})
#GAN network
convolution_param = dict(num_output=16, kernel_size=3, stride=1, pad=1, weight_filler = dict(type='xavier'))
n.ganAconv1 = L.Convolution(n.Ainp, convolution_param=convolution_param)
n.ganAconv1 = L.ReLU(n.ganAconv1, negative_slope=0.1)
convolution_param = dict(num_output=16, kernel_size=3, stride=1, pad=1, weight_filler = dict(type='xavier'))
n.ganAconv2 = L.Convolution(n.ganAconv1, convolution_param=convolution_param)
n.ganAconv2 = L.ReLU(n.ganAconv2, negative_slope=0.1)
convolution_param = dict(num_output=32, kernel_size=2, stride=2, pad=0, weight_filler = dict(type='xavier'))
n.ganAconv3 = L.Convolution(n.ganAconv2, convolution_param=convolution_param)
n.ganAconv3 = L.ReLU(n.ganAconv3, negative_slope=0.1)
convolution_param = dict(num_output=32, kernel_size=3, stride=1, pad=1, weight_filler = dict(type='xavier'))
n.ganAconv4 = L.Convolution(n.ganAconv3, convolution_param=convolution_param)
n.ganAconv4 = L.ReLU(n.ganAconv4, negative_slope=0.1)
convolution_param = dict(num_output=32, kernel_size=3, stride=1, pad=1, weight_filler = dict(type='xavier'))
n.ganAconv5 = L.Convolution(n.ganAconv4, convolution_param=convolution_param)
#n.ganAconv5 = L.BatchNorm(n.ganAconv5, use_global_stats=global_stat, name=allparamnames.pop(0))#, param=[{"lr_mult":0},{"lr_mult":0},{"lr_mult":0}])
n.ganAconv5 = L.ReLU(n.ganAconv5, negative_slope=0.1)
convolution_param = dict(num_output=32, kernel_size=2, stride=2, pad=0, weight_filler = dict(type='xavier'))
n.ganAconv6 = L.Convolution(n.ganAconv5, convolution_param=convolution_param)
#n.ganAconv6 = L.BatchNorm(n.ganAconv6, use_global_stats=global_stat, name=allparamnames.pop(0))#, param=[{"lr_mult":0},{"lr_mult":0},{"lr_mult":0}])
n.ganAconv6 = L.ReLU(n.ganAconv6, negative_slope=0.1)
convolution_param = dict(num_output=32, kernel_size=1, stride=1, pad=0, weight_filler = dict(type='xavier'))
n.ganAconv7 = L.Convolution(n.ganAconv6, convolution_param=convolution_param)
#n.ganAconv7 = L.BatchNorm(n.ganAconv7, use_global_stats=global_stat, name=allparamnames.pop(0))#, param=[{"lr_mult":0},{"lr_mult":0},{"lr_mult":0}])
n.ganAconv7 = L.ReLU(n.ganAconv7, negative_slope=0.1)
n.ganAconv7_pool = L.Pooling(n.ganAconv7, global_pooling=True, pool=P.Pooling.AVE)
n.Aip3 = L.InnerProduct(n.ganAconv7_pool, num_output=1, weight_filler=dict(type='xavier'), name="last")
n.Aloss = L.SigmoidCrossEntropyLoss(n.Aip3, n.Alabels)
return n
def base_ae(src_train, src_test):
n = caffe.NetSpec()
n.data = \
L.Data(batch_size=100,
backend=P.Data.LMDB,
source=src_train,
transform_param=dict(scale=0.0039215684),
ntop=1,
include=[dict(phase=caffe.TRAIN)]
)
n.data_test = \
L.Data(batch_size=100, backend=P.Data.LMDB,
source=src_test,
transform_param=dict(scale=0.0039215684),
ntop=1,
include=[dict(phase=caffe.TEST)]
)
n.flatdata = L.Flatten(n.data)
n.encode001 = \
L.InnerProduct(n.data,
num_output=64,
param=[dict(lr_mult=1, decay_mult=1),
dict(lr_mult=1, decay_mult=0)],
weight_filler=dict(type='gaussian', std=1, sparse=15),
bias_filler=dict(type='constant', value=0)
)
n.encode001neuron = L.Sigmoid(n.encode001, in_place=True)
n.decode001 = L.InnerProduct(
n.encode001neuron,
num_output=3072,
param=[dict(lr_mult=1, decay_mult=1),
dict(lr_mult=1, decay_mult=0)],
weight_filler=dict(type='gaussian', std=1, sparse=15),
bias_filler=dict(type='constant', value=0))
n.loss_x_entropy = \
L.SigmoidCrossEntropyLoss(n.decode001, n.flatdata,
loss_weight=[1])
n.decode001neuron = L.Sigmoid(n.decode001, in_place=False)
n.loss_l2 = \
L.EuclideanLoss(n.decode001neuron, n.flatdata,
loss_weight=[0])
n_proto = n.to_proto()
# fix data layer for test phase
for l in n_proto.layer:
if l.type.lower() == 'data' and \
[x.phase for x in l.include] == [caffe.TEST]:
for t in list(l.top):
l.top.remove(t)
t = t.split('_test')[0]
l.top.append(unicode(t))
l.name = l.name.split('_test')[0]
return n_proto
def build_AlexNet_2heads(split,
num_classes,
batch_size,
resize_w,
resize_h,
crop_w=0,
crop_h=0,
crop_margin=0,
mirror=0,
rotate=0,
HSV_prob=0,
HSV_jitter=0,
train=True,
deploy=False):
weight_param = dict(lr_mult=1, decay_mult=1)
bias_param = dict(lr_mult=2, decay_mult=0)
learned_param = [weight_param, bias_param]
frozen_param = [dict(lr_mult=0)] * 2
n = caffe.NetSpec()
pydata_params = dict(split=split, mean=(104.00699, 116.66877, 122.67892))
pydata_params['dir'] = '../../../datasets/SocialMedia'
pydata_params['train'] = train
pydata_params['batch_size'] = batch_size
pydata_params['resize_w'] = resize_w
pydata_params['resize_h'] = resize_h
pydata_params['crop_w'] = crop_w
pydata_params['crop_h'] = crop_h
pydata_params['crop_margin'] = crop_margin
pydata_params['mirror'] = mirror
pydata_params['rotate'] = rotate
pydata_params['HSV_prob'] = HSV_prob
pydata_params['HSV_jitter'] = HSV_jitter
pydata_params['num_classes'] = num_classes
pylayer = 'twoHeadsDataLayer'
n.data, n.label, n.label_class = L.Python(module='layers_2heads',
layer=pylayer,
ntop=3,
param_str=str(pydata_params))
# Convs
n.conv1, n.relu1 = conv_relu(n.data, 11, 96, stride=4, param=learned_param)
n.pool1 = max_pool(n.relu1, 3, stride=2)
n.norm1 = L.LRN(n.pool1, local_size=5, alpha=1e-4, beta=0.75)
n.conv2, n.relu2 = conv_relu(n.norm1,
5,
256,
pad=2,
group=2,
param=learned_param)
n.pool2 = max_pool(n.relu2, 3, stride=2)
n.norm2 = L.LRN(n.pool2, local_size=5, alpha=1e-4, beta=0.75)
n.conv3, n.relu3 = conv_relu(n.norm2, 3, 384, pad=1, param=learned_param)
n.conv4, n.relu4 = conv_relu(n.relu3,
3,
384,
pad=1,
group=2,
param=learned_param)
n.conv5, n.relu5 = conv_relu(n.relu4,
3,
256,
pad=1,
group=2,
param=learned_param)
n.pool5 = max_pool(n.relu5, 3, stride=2)
# Regression head
n.fc6, n.relu6 = fc_relu(n.pool5, 4096, param=learned_param) #4096
if train:
n.drop6 = fc7input = L.Dropout(n.relu6,
in_place=True,
dropout_ratio=0.5)
else:
fc7input = n.relu6
n.fc7, n.relu7 = fc_relu(fc7input, 4096, param=learned_param) #4096
if train:
n.drop7 = fc8input = L.Dropout(n.relu7,
in_place=True,
dropout_ratio=0.5)
else:
fc8input = n.relu7
n.fc8C = L.InnerProduct(fc8input,
num_output=num_classes,
weight_filler=dict(type='gaussian', std=0.01),
bias_filler=dict(type='constant', value=0.1),
param=learned_param)
# Regression loss
if not deploy:
n.loss = L.SigmoidCrossEntropyLoss(n.fc8C, n.label, loss_weight=1)
# Classification head
n.fc6_class, n.relu6_class = fc_relu(n.pool5, 4096,
param=learned_param) #4096
if train:
n.drop6_class = fc7input_class = L.Dropout(n.relu6_class,
in_place=True,
dropout_ratio=0.5)
else:
fc7input_class = n.relu6_class
n.fc7_class, n.relu7_class = fc_relu(fc7input_class,
4096,
param=learned_param) #4096
if train:
n.drop7_class = fc8input_class = L.Dropout(n.relu7_class,
in_place=True,
dropout_ratio=0.5)
else:
fc8input_class = n.relu7_class
n.fc8_class = L.InnerProduct(fc8input_class,
num_output=10,
weight_filler=dict(type='gaussian', std=0.01),
bias_filler=dict(type='constant', value=0.1),
param=learned_param)
# Classification loss and acc
if not deploy:
n.loss_class = L.SoftmaxWithLoss(n.fc8_class,
n.label_class,
loss_weight=0.3)
n.acc_class = L.Accuracy(n.fc8_class, n.label_class)
# Deploy output processing
if deploy:
n.probs = L.Sigmoid(n.fc8C)
n.probs_class = L.Softmax(n.fc8_class)
with open('deploy.prototxt', 'w') as f:
f.write(str(n.to_proto()))
return f.name
else:
if train:
with open('train.prototxt', 'w') as f:
f.write(str(n.to_proto()))
return f.name
else:
with open('val.prototxt', 'w') as f:
f.write(str(n.to_proto()))
return f.name
def resnet_mask_end2end(self):
channals = self.channals
if not self.deploy:
data, im_info, gt_boxes, ins = \
data_layer_train_with_ins(self.net, self.classes, with_rpn=True)
else:
data, im_info = data_layer_test(self.net)
gt_boxes = None
conv1 = conv_factory(self.net, "conv1", data, 7, channals, 2, 3, bias_term=True)
pool1 = pooling_layer(self.net, 3, 2, 'MAX', 'pool1', conv1)
index = 1
out = pool1
if self.module == "normal":
residual_block = residual_block
else:
residual_block = residual_block_basic
for i in self.stages[:-1]:
index += 1
for j in range(i):
if j == 0:
if index == 2:
stride = 1
else:
stride = 2
out = residual_block(self.net, "res" + str(index) + ascii_lowercase[j], out, channals, stride)
else:
out = residual_block(self.net, "res" + str(index) + ascii_lowercase[j], out, channals)
channals *= 2
if not self.deploy:
rpn_cls_loss, rpn_loss_bbox, rpn_cls_score_reshape, rpn_bbox_pred = rpn(self.net, out, gt_boxes, im_info, data, fixed=False)
rois, labels, bbox_targets, bbox_inside_weights, bbox_outside_weights, mask_roi, masks = \
roi_proposals(self.net, rpn_cls_score_reshape, rpn_bbox_pred, im_info, gt_boxes)
self.net["rois_cat"] = L.Concat(rois,mask_roi, name="rois_cat", axis=0)
rois=self.net["rois_cat"]
else:
rpn_cls_score_reshape, rpn_bbox_pred = rpn(self.net, out, gt_boxes, im_info, data)
rois, scores = roi_proposals(self.net, rpn_cls_score_reshape, rpn_bbox_pred, im_info, gt_boxes)
feat_out = out
feat_aligned = roi_align(self.net, "det_mask", feat_out, rois)
# if not self.deploy:
# self.net["silence_mask_rois"] = L.Silence(mask_rois, ntop=0)
# if not self.deploy:
# mask_feat_aligned = self.roi_align("mask", feat_out, mask_rois)
# else:
# mask_feat_aligned = self.roi_align("mask", feat_out, rois)
out = feat_aligned
index += 1
for j in range(self.stages[-1]):
if j == 0:
stride = 1
out = residual_block(self.net, "res" + str(index) + ascii_lowercase[j], out, channals, stride)
else:
out = residual_block(self.net, "res" + str(index) + ascii_lowercase[j], out, channals)
if not self.deploy:
self.net["det_feat"], self.net["mask_feat"] = L.Slice(self.net, out, ntop=2, name='slice', slice_param=dict(slice_dim=0, slice_point=self.rois_num))
feat_mask = self.net["mask_feat"]
out = self.net["det_feat"]
# for bbox detection
pool5 = ave_pool(self.net, 7, 1, "pool5", out)
cls_score, bbox_pred = final_cls_bbox(self.net, pool5)
if not self.deploy:
self.net["loss_cls"] = L.SoftmaxWithLoss(cls_score, labels, loss_weight=1, propagate_down=[1, 0])
self.net["loss_bbox"] = L.SmoothL1Loss(bbox_pred, bbox_targets, bbox_inside_weights, bbox_outside_weights, \
loss_weight=1)
else:
self.net["cls_prob"] = L.Softmax(cls_score)
# # for mask prediction
if not self.deploy:
mask_feat_aligned = feat_mask
else:
mask_feat_aligned = out
# out = mask_feat_aligned
out = L.Deconvolution(mask_feat_aligned, name = "mask_deconv1",convolution_param=dict(kernel_size=2, stride=2,
num_output=256, pad=0, bias_term=False,
weight_filler=dict(type='msra')))
out = L.BatchNorm(out, name="bn_mask_deconv1",in_place=True, batch_norm_param=dict(use_global_stats=self.deploy))
out = L.Scale(out, name = "scale_mask_deconv1", in_place=True, scale_param=dict(bias_term=True))
out = L.ReLU(out, name="mask_deconv1_relu", in_place=True)
mask_out = conv_factory(self.net, "mask_out", out, 1, self.classes-1, 1, 0, bias_term=True)
# for i in range(4):
# out = self.conv_factory("mask_conv"+str(i), out, 3, 256, 1, 1, bias_term=False)
# mask_out = self.conv_factory("mask_out", out, 1, 1, 1, 0, bias_term=False)
if not self.deploy:
self.net["loss_mask"] = L.SigmoidCrossEntropyLoss(mask_out, masks, loss_weight=1, propagate_down=[1, 0],
loss_param=dict(
normalization=1,
ignore_label = -1
))
else:
self.net["mask_prob"] = L.Sigmoid(mask_out)
return self.net.to_proto()
