def train_head(self, subset):
n = NetSpec()
# train
image_data_param = dict(source=subset.get_list_absolute_path(),
batch_size=self.batch_sizes[0],
new_width=self.infmt.new_width,
new_height=self.infmt.new_height,
rand_skip=self.batch_size,
shuffle=True)
if subset.root_folder is not None:
image_data_param['root_folder'] = subset.root_folder
transform_param = dict(
mirror=self.infmt.mirror,
crop_size=self.infmt.crop_size,
# mean_value = self.infmt.mean_pixel,
)
if self.infmt.scale is not None:
transform_param['scale'] = self.infmt.scale
if self.infmt.mean_file is not None:
transform_param['mean_file'] = self.infmt.mean_file
elif self.infmt.mean_pixel is not None:
transform_param['mean_value'] = self.infmt.mean_pixel
n.data, n.label = L.ImageData(ntop=2,
image_data_param=image_data_param,
transform_param=transform_param,
include=dict(phase=caffe.TRAIN))
net = n.to_proto()
net.name = self.name
return net
def main():
from argparse import ArgumentParser
from os import path
parser = ArgumentParser()
parser.add_argument('prototxt')
parser.add_argument('-l', '--load', help='Load a caffemodel')
parser.add_argument('-d', '--data', default=None, help='Image list to use [default prototxt data]')
#parser.add_argument('-q', action='store_true', help='Quiet execution')
parser.add_argument('-sm', action='store_true', help='Summary only')
parser.add_argument('-q', action='store_true', help='Quiet execution')
parser.add_argument('-a', '--all', action='store_true', help='Show the statistic for all layers')
parser.add_argument('-nc', action='store_true', help='Do not use color')
parser.add_argument('-s', type=float, default=1.0, help='Scale the input [only custom data "-d"]')
parser.add_argument('-bs', type=int, default=16, help='Batch size [only custom data "-d"]')
parser.add_argument('-nit', type=int, default=10, help='Number of iterations')
parser.add_argument('--gpu', type=int, default=0, help='What gpu to run it on?')
args = parser.parse_args()
if args.q:
from os import environ
environ['GLOG_minloglevel'] = '2'
import caffe, load
from caffe import NetSpec, layers as L
caffe.set_mode_gpu()
if args.gpu is not None:
caffe.set_device(args.gpu)
model = load.ProtoDesc(args.prototxt)
net = NetSpec()
if args.data is not None:
fl = getFileList(args.data)
if len(fl) == 0:
print("Unknown data type for '%s'"%args.data)
exit(1)
from tempfile import NamedTemporaryFile
f = NamedTemporaryFile('w')
f.write('\n'.join([path.abspath(i)+' 0' for i in fl]))
f.flush()
net.data, net.label = L.ImageData(source=f.name, batch_size=args.bs, new_width=model.input_dim[-1], new_height=model.input_dim[-1], transform_param=dict(mean_value=[104,117,123], scale=args.s),ntop=2)
net.out = model(data=net.data, label=net.label)
else:
net.out = model()
n = netFromString('force_backward:true\n'+str(net.to_proto()), caffe.TRAIN )
if args.load is not None:
n.copy_from(args.load)
cvar = printMeanStddev(n, NIT=args.nit, show_all=args.all, show_color=not args.nc, quiet=args.sm)
cv, gr = computeGraidentRatio(n, NIT=args.nit)
print()
print(' Summary ')
print('-----------')
print()
print('layer name out cvar rate cvar rate mean')
for l in n._layer_names:
if l in cvar and l in cv and l in gr:
print('%-30s %10.2f %10.2f %10.2f'%(l, cvar[l], cv[l], gr[l]) )
def get_phocnet(self, word_image_lmdb_path, phoc_lmdb_path,
phoc_size=604, generate_deploy=False):
'''
Returns a NetSpec definition of the PHOCNet. The definition can then be transformed
into a protobuffer message by casting it into a str.
'''
n = NetSpec()
# Data
self.set_phocnet_data(n=n, generate_deploy=generate_deploy,
word_image_lmdb_path=word_image_lmdb_path,
phoc_lmdb_path=phoc_lmdb_path)
# Conv Part
self.set_phocnet_conv_body(n=n, relu_in_place=True)
# FC Part
n.spp5 = L.SPP(n.relu4_3, spp_param=dict(pool=P.SPP.MAX, pyramid_height=3, engine=self.spp_engine))
n.fc6, n.relu6, n.drop6 = self.fc_relu(bottom=n.spp5, layer_size=4096,
dropout_ratio=0.5, relu_in_place=True)
n.fc7, n.relu7, n.drop7 = self.fc_relu(bottom=n.drop6, layer_size=4096,
dropout_ratio=0.5, relu_in_place=True)
n.fc8 = L.InnerProduct(n.drop7, num_output=phoc_size,
weight_filler=dict(type=self.initialization),
bias_filler=dict(type='constant'))
n.sigmoid = L.Sigmoid(n.fc8, include=dict(phase=self.phase_test))
# output part
if not generate_deploy:
n.silence = L.Silence(n.sigmoid, ntop=0, include=dict(phase=self.phase_test))
n.loss = L.SigmoidCrossEntropyLoss(n.fc8, n.phocs)
return n.to_proto()
def __init__(self, number_of_neighbors=6, inner_product_output=100, weight_lr_mult=1, weight_decay_mult=1, b_lr_mult=2, b_decay_mult=0): self.number_of_neighbors = number_of_neighbors #self.netP = self.net = NetSpec() self.shared_weight_counter = 0 self.num_output = inner_product_output self.weight_lr_mult = weight_lr_mult self.weight_decay_mult = weight_decay_mult self.b_lr_mult = b_lr_mult self.b_decay_mult = b_decay_mult
def val_tail(self, last_top, stage=None):
n = NetSpec()
include_param = dict(phase=caffe.TEST)
if stage is not None:
include_param['stage'] = stage
if stage is None:
n.loss = L.SoftmaxWithLoss(bottom=[last_top, "label"])
n.accuracy = L.Accuracy(bottom=[last_top, "label"],
include=include_param)
return n.to_proto()
def val_head(self, subset, stage=None):
image_data_param = dict(
source=subset.get_list_absolute_path(),
batch_size=self.batch_sizes[1],
# root_folder=subset.root_folder,
rand_skip=self.batch_sizes[1],
shuffle=True,
# new_width,
# new_height
)
transform_param = dict(
mirror=False,
# crop_size = self.infmt.crop_size,
# mean_value = self.infmt.mean_pixel,
# mean_file,
# scale,
)
if subset.root_folder is not None:
image_data_param['root_folder'] = subset.root_folder
if self.crop_on_test:
image_data_param['new_width'] = self.infmt.new_width
image_data_param['new_height'] = self.infmt.new_height
transform_param['crop_size'] = self.infmt.crop_size
else:
image_data_param['new_width'] = self.infmt.crop_size
image_data_param['new_height'] = self.infmt.crop_size
if self.infmt.scale is not None:
transform_param['scale'] = self.infmt.scale
if self.infmt.mean_file is not None:
transform_param['mean_file'] = self.infmt.mean_file
elif self.infmt.mean_pixel is not None:
transform_param['mean_value'] = self.infmt.mean_pixel
include_param = dict(phase=caffe.TEST)
if stage is not None:
include_param['stage'] = stage
n = NetSpec()
n.data, n.label = L.ImageData(ntop=2,
image_data_param=image_data_param,
transform_param=transform_param,
include=include_param)
net = n.to_proto()
net.name = self.name
return net
def __init__(self,
data_layer_params,
datalayer='FlowDatalayer',
deploy=False):
# setup the python data layer
self.net = NetSpec()
if deploy:
batch_size = data_layer_params['batch_size']
shape = data_layer_params['im_shape']
input_param=dict(shape=dict(dim=[batch_size,6,shape[1],shape[0]]))
self.net.data = L.Input(ntop=1, input_param=input_param)
else:
self.net.data, self.net.flow = L.Python(module='flow_datalayer',
layer=datalayer,
ntop=2,
param_str=str(data_layer_params))
self.net.gt = L.Pooling(self.net.flow, pool=P.Pooling.AVE, kernel_size=4, stride=4)
self.net.flat_gt = L.Reshape(self.net.gt, reshape_param=dict(shape=dict(dim=[-1,1,1,1])))
# the net itself
self.net.conv1, self.net.relu1 = conv_relu(self.net.data, 7, 64, stride=1, pad=3)
self.net.pool1 = max_pool(self.net.relu1, 3, stride=2)
self.net.conv2, self.net.relu2 = conv_relu(self.net.pool1, 5, 128, pad=2)
self.net.pool2 = max_pool(self.net.relu2, 3, stride=2)
self.net.conv3, self.net.relu3 = conv_relu(self.net.pool2, 3, 384, pad=1)
self.net.conv4, self.net.relu4 = conv_relu(self.net.relu3, 3, 384, pad=1)
self.net.conv5, self.net.relu5 = conv_relu(self.net.relu4, 3, 256, pad=1)
self.net.flow_est = L.Convolution(self.net.relu5, kernel_size=1, num_output=2,
weight_filler=dict(type='xavier'),
bias_filler=dict(type='constant', value=0))
if not deploy:
self.net.flat_est = L.Reshape(self.net.flow_est, reshape_param=dict(shape=(dict(dim=[-1,1,1,1]))))
self.net.loss = L.EuclideanLoss(self.net.flat_est, self.net.flat_gt)
def extract_head(self, subset):
image_data_param = dict(
source=subset.get_list_absolute_path(),
batch_size=self.batch_size,
root_folder=subset.root_folder,
# new_width,
# new_height
)
transform_param = dict(
mirror=False,
# crop_size = self.infmt.crop_size,
# mean_value = self.infmt.mean_pixel,
# mean_file,
# scale,
)
if self.crop_on_test:
image_data_param['new_width'] = self.infmt.new_width
image_data_param['new_height'] = self.infmt.new_height
transform_param['crop_size'] = self.infmt.crop_size
else:
image_data_param['new_width'] = self.infmt.crop_size
image_data_param['new_height'] = self.infmt.crop_size
if self.infmt.scale is not None:
transform_param['scale'] = self.infmt.scale
if self.infmt.mean_file is not None:
transform_param['mean_file'] = self.infmt.mean_file
elif self.infmt.mean_pixel is not None:
transform_param['mean_value'] = self.infmt.mean_pixel
n = NetSpec()
n.data, n.label = L.ImageData(ntop=2,
image_data_param=image_data_param,
transform_param=transform_param
) # , include=dict(phase=caffe.TEST))
net = n.to_proto()
net.name = self.name
return net
def NiN(opts):
n = NetSpec()
FireNet_data_layer(n, batch_size) #add data layer to the net
curr_bottom = 'data'
#TODO: possibly rename layers to conv1.1, 1.2, 1.3; 2.1, 2.2, etc.
curr_bottom = conv_relu_xavier(n, 11, 96, str(1), 4, 0, curr_bottom) #_, ksize, nfilt, layerIdx, stride, pad, _
if 'pool1' in opts:
curr_bottom = NiN_pool(n, str(3), curr_bottom)
curr_bottom = conv_relu_xavier(n, 1, 96, str(2), 1, 0, curr_bottom)
curr_bottom = conv_relu_xavier(n, 1, 96, str(3), 1, 0, curr_bottom)
curr_bottom = NiN_pool(n, str(3), curr_bottom)
curr_bottom = conv_relu_xavier(n, 5, 256, str(4), 1, 2, curr_bottom)
curr_bottom = conv_relu_xavier(n, 1, 256, str(5), 1, 0, curr_bottom)
curr_bottom = conv_relu_xavier(n, 1, 256, str(6), 1, 0, curr_bottom)
curr_bottom = NiN_pool(n, str(6), curr_bottom)
#conv8 and conv9 are the least computationally intensive layers
curr_bottom = conv_relu_xavier(n, 3, 384, str(7), 1, 1, curr_bottom)
conv8_nfilt = get_conv8_nfilt(opts)
curr_bottom = conv_relu_xavier(n, 1, conv8_nfilt, str(8), 1, 0, curr_bottom)
curr_bottom = conv_relu_xavier(n, 1, 384, str(9), 1, 0, curr_bottom)
curr_bottom = NiN_pool(n, str(9), curr_bottom)
n.tops['drop9'] = L.Dropout(n.tops[curr_bottom], dropout_ratio=0.5, in_place=True)
curr_bottom = conv_relu_xavier(n, 3, 1024, str(10), 1, 1, curr_bottom)
curr_bottom = conv_relu_xavier(n, 1, 1024, str(11), 1, 0, curr_bottom)
num_output=1000
if 'out10k' in opts:
num_output=10000
n.tops['conv_12'] = L.Convolution(n.tops[curr_bottom], kernel_size=1, num_output=num_output, weight_filler=dict(type='gaussian', std=0.01, mean=0.0))
n.tops['relu_conv_12'] = L.ReLU(n.tops['conv_12'], in_place=True)
n.tops['pool_12'] = L.Pooling(n.tops['conv_12'], global_pooling=1, pool=P.Pooling.AVE)
if phase == 'trainval':
n.loss = L.SoftmaxWithLoss(n.tops['pool_12'], n.label, include=dict(phase=caffe_pb2.TRAIN))
n.accuracy = L.Accuracy(n.tops['pool_12'], n.label, include=dict(phase=caffe_pb2.TEST))
n.accuracy_top5 = L.Accuracy(n.tops['pool_12'], n.label, include=dict(phase=caffe_pb2.TEST), top_k=5)
out_dir = 'nets/NiN_' + '_'.join(opts)
return [n.to_proto(), out_dir]
class Network:
def __init__(self,
number_of_neighbors=6,
inner_product_output=100,
weight_lr_mult=1,
weight_decay_mult=1,
b_lr_mult=2,
b_decay_mult=0,
feature_type=[]):
self.number_of_neighbors = number_of_neighbors
#self.netP =
self.net = NetSpec()
self.shared_weight_counter = 0
self.num_output = inner_product_output
self.weight_lr_mult = weight_lr_mult
self.weight_decay_mult = weight_decay_mult
self.b_lr_mult = b_lr_mult
self.b_decay_mult = b_decay_mult
self.feature_type = [x.name for x in feature_type]
def getInnerProduct(self, input_name, output_name, ID, num_output=None):
#TODO What should be the output of this layer?
if num_output is None:
num_output = self.num_output
return L.InnerProduct(
getattr(self.net, input_name),
name=output_name,
weight_filler=dict(type='xavier'),
bias_filler=dict(type='xavier', value=0.2),
num_output=num_output,
#in_place=True,
param=list([
dict(name="embed_w{0}".format(ID),
lr_mult=self.weight_lr_mult,
decay_mult=self.weight_decay_mult),
dict(name="embed_b{0}".format(ID),
lr_mult=self.b_lr_mult,
decay_mult=self.b_decay_mult)
]))
def getInnerProduct_with_relu(self,
input_name,
output_name,
ID,
num_output=None):
#TODO What should be the output of this layer?
if num_output is None:
num_output = self.num_output
lay = L.InnerProduct(
getattr(self.net, input_name),
name=output_name,
weight_filler=dict(type='xavier'),
bias_filler=dict(type='xavier', value=0.2),
num_output=num_output,
#in_place=True,
param=list([
dict(name="embed_w{0}".format(ID),
lr_mult=self.weight_lr_mult,
decay_mult=self.weight_decay_mult),
dict(name="embed_b{0}".format(ID),
lr_mult=self.b_lr_mult,
decay_mult=self.b_decay_mult)
]))
setattr(self.net, output_name, lay)
return L.ReLU(lay,
name=output_name.replace('inner_product', 'relu'),
in_place=True)
def return_layer(self, feature_key, layer_key):
#layer = L.InnerProductLayer .....
setattr(self.net,
'inner_product_layer_{0}_{1}'.format(feature_key,
layer_key), layer)
return
def createEmbeddingNetwork(self,
database_list_path='.',
batch_size=20,
phase=0):
dataset_path = database_list_path
dataLayer = L.HDF5Data(
name='dataLayer',
source=dataset_path,
batch_size=batch_size,
ntop=len(self.feature_type) * (2 + self.number_of_neighbors),
include=list([dict(phase=phase)
])) # tops-> target, [neighbors], negative
#data -> [target, neighbor1, neighbor2, ..., neighbork, negative]
feature_type = self.feature_type
for i in xrange(len(self.feature_type)):
start = (i - 1) * (len(dataLayer) / len(self.feature_type))
end = i * (len(dataLayer) / len(self.feature_type)) - 1
setattr(self.net, '{0}_target'.format(feature_type[i]),
dataLayer[start])
setattr(self.net, '{0}_negative'.format(feature_type[i]),
dataLayer[end])
for l in range(start + 1, end):
setattr(
self.net, '{0}_neighbor{1}'.format(feature_type[i],
l - start - 1),
dataLayer[l])
#First layer of inner product
layer = self.getInnerProduct_with_relu(
'{0}_target'.format(feature_type[i]),
'{0}_inner_product_target_1'.format(feature_type[i]),
'_{}_2'.format(self.feature_type[i]),
num_output=1000)
setattr(self.net, '{0}_relu_target_1'.format(feature_type[i]),
layer)
layer = self.getInnerProduct_with_relu(
'{0}_negative'.format(feature_type[i]),
'{0}_inner_product_negative_1'.format(feature_type[i]),
'_{}_2'.format(self.feature_type[i]),
num_output=1000)
setattr(self.net, '{0}_relu_negative_1'.format(feature_type[i]),
layer)
for j in range(self.number_of_neighbors):
layer = self.getInnerProduct_with_relu(
'{0}_neighbor{1}'.format(feature_type[i], j),
'{0}_inner_product_neighbor{1}_1'.format(
feature_type[i], j),
'_{}_2'.format(self.feature_type[i]),
num_output=1000)
setattr(self.net,
'{0}_relu_neighbor{1}_1'.format(feature_type[i],
j), layer)
#Relu on top of the fisrt inner product
concat_inputs = ['{0}_relu'.format(x) for x in feature_type]
self.net.inner_product_target_1 = L.Concat(
*map(lambda x: getattr(self.net, x + '_target_1'), concat_inputs),
name='inner_product_target_1',
axis=1)
self.net.inner_product_negative_1 = L.Concat(
*map(lambda x: getattr(self.net, x + '_negative_1'),
concat_inputs),
name='inner_product_negative_1',
axis=1)
for i in range(self.number_of_neighbors):
layer = L.Concat(*map(
lambda x: getattr(self.net, x + '_neighbor{0}_1'.format(i)),
concat_inputs),
name='inner_product_neighbor{0}_1'.format(i),
axis=1)
setattr(self.net, 'inner_product_neighbor{0}_1'.format(i), layer)
#for i in xrange(len(self.feature_type)):
# self.inner_product_target1 =
#Second layer of inner product
self.net.inner_product_target = self.getInnerProduct_with_relu(
'inner_product_target_1', 'inner_product_target', 1)
self.net.inner_product_negative = self.getInnerProduct_with_relu(
'inner_product_negative_1', 'inner_product_negative', 1)
for i in range(0, self.number_of_neighbors):
layer = self.getInnerProduct_with_relu(
'inner_product_neighbor{0}_1'.format(i),
'inner_product_neighbor{0}'.format(i), 1)
setattr(self.net, 'relu_neighbor{0}'.format(i), layer)
#self.net.normalize_target = L.NormalizeLayer(self.net.inner_product_target, name='normalize_target', in_place=True)
#self.net.normalize_negative = L.NormalizeLayer(self.net.inner_product_negative, name='normalize_negative', in_place=True)
#for i in range(0, self.number_of_neighbors):
# layer = L.NormalizeLayer(getattr(self.net, 'inner_product_neighbor{0}'.format(i)),
# name='normalize_neighbor{0}'.format(i),
# in_place=True)
# setattr(self.net, 'normalize_neighbor{0}'.format(i), layer)
#Second layer of inner product
#self.net.inner_product2_target = self.getInnerProduct('inner_product_target', 'inner_product2_target', 2)
#self.net.inner_product2_negative = self.getInnerProduct('inner_product_negative', 'inner_product2_negative', 2)
#for i in range(0, self.number_of_neighbors):
# layer = self.getInnerProduct('inner_product_neighbor{0}'.format(i),
# 'inner_product2_neighbor{0}'.format(i), 2)
# setattr(self.net, 'inner_product2_neighbor{0}'.format(i), layer)
#Context
'''
context_sum_bottom = []
for i in range(0, self.number_of_neighbors):
context_sum_bottom.append(getattr(self.net, 'inner_product2_neighbor{0}'.format(i)))
coeff = 1.0/self.number_of_neighbors
self.net.context_sum = L.Eltwise(*context_sum_bottom,
name='context_sum',
operation=P.Eltwise.SUM, # 1 -> SUM
coeff=list([coeff for i in range(self.number_of_neighbors)]))
#Target - Negative
self.net.target_negative_diff = L.Eltwise(self.net.inner_product2_target, self.net.inner_product2_negative,
name='target_negative_diff',
operation=P.Eltwise.SUM, # SUM
coeff=list([1,-1])) # target - negative
'''
#Context
context_sum_bottom = []
for i in range(0, self.number_of_neighbors):
context_sum_bottom.append(
getattr(self.net, 'inner_product_neighbor{0}'.format(i)))
coeff = 1.0 / self.number_of_neighbors
self.net.context_sum = L.Eltwise(
*context_sum_bottom,
name='context_sum',
operation=P.Eltwise.SUM, # SUM
coeff=list([coeff for i in range(self.number_of_neighbors)]))
#self.net.
#Target - Negative
self.net.target_negative_diff = L.Eltwise(
self.net.inner_product_target,
self.net.inner_product_negative,
name='target_negative_diff',
operation=P.Eltwise.SUM, # SUM
coeff=list([1, -1])) # target - negative
#Loss layer
self.net.loss = L.Python(self.net.context_sum,
self.net.target_negative_diff,
name='loss',
module='my_dot_product_layer',
layer='MyHingLossDotProductLayer')
def saveNetwork(self, model_prototxt_path):
with open(model_prototxt_path, 'w') as f:
f.write("force_backward:true\n" + str(self.net.to_proto()))
class Network:
def __init__(self, number_of_neighbors, num_output=100, lr_mult):
self.number_of_neighbors = number_of_neighbors
#self.netP =
self.net = NetSpec()
self.shared_weight_counter = 0
self.num_output = num_output
def getInnerProduct(self, input_name, output_name, ID, num_output=None):
#TODO What should be the output of this layer?
if num_output is None:
num_output = self.num_output
return L.InnerProduct(
getattr(self.net, input_name),
name=output_name,
weight_filler=dict(type='xavier'),
bias_filler=dict(type='xavier', value=0.2),
num_output=num_output,
#in_place=True,
param=list([
dict(name="embed_w{0}".format(ID), lr_mult=1, decay_mult=1),
dict(name="embed_b{0}".format(ID), lr_mult=2, decay_mult=0)
]))
def createEmbeddingNetwork(self, dataset_path, batch_size):
dataLayer = L.HDF5Data(
name='dataLayer',
source=dataset_path,
batch_size=batch_size,
ntop=2 +
self.number_of_neighbors) # tops-> target, [neighbors], negative
#data -> [target, neighbor1, neighbor2, ..., neighbork, negative]
self.net.target = dataLayer[0]
self.net.negative = dataLayer[-1]
for l in range(1, self.number_of_neighbors + 1):
setattr(self.net, 'neighbor{0}'.format(l - 1), dataLayer[l])
#First layer of inner product
self.net.inner_product_target = self.getInnerProduct(
'target', 'inner_product_target', 1)
self.net.inner_product_negative = self.getInnerProduct(
'negative', 'inner_product_negative', 1)
for i in range(0, self.number_of_neighbors):
layer = self.getInnerProduct('neighbor{0}'.format(i),
'inner_product_neighbor{0}'.format(i),
1)
setattr(self.net, 'inner_product_neighbor{0}'.format(i), layer)
#ReLU
self.net.relu_target = L.ReLU(self.net.inner_product_target,
name='relu_target',
in_place=True)
self.net.relu_negative = L.ReLU(self.net.inner_product_negative,
name='relu_negative',
in_place=True)
for i in range(0, self.number_of_neighbors):
layer = L.ReLU(getattr(self.net,
'inner_product_neighbor{0}'.format(i)),
name='relu_neighbor{0}'.format(i),
in_place=True)
setattr(self.net, 'relu_neighbor{0}'.format(i), layer)
#Second layer of inner product
#self.net.inner_product2_target = self.getInnerProduct('inner_product_target', 'inner_product2_target', 2)
#self.net.inner_product2_negative = self.getInnerProduct('inner_product_negative', 'inner_product2_negative', 2)
#for i in range(0, self.number_of_neighbors):
# layer = self.getInnerProduct('inner_product_neighbor{0}'.format(i),
# 'inner_product2_neighbor{0}'.format(i), 2)
# setattr(self.net, 'inner_product2_neighbor{0}'.format(i), layer)
#Context
'''
context_sum_bottom = []
for i in range(0, self.number_of_neighbors):
context_sum_bottom.append(getattr(self.net, 'inner_product2_neighbor{0}'.format(i)))
coeff = 1.0/self.number_of_neighbors
self.net.context_sum = L.Eltwise(*context_sum_bottom,
name='context_sum',
operation=P.Eltwise.SUM, # 1 -> SUM
coeff=list([coeff for i in range(self.number_of_neighbors)]))
#Target - Negative
self.net.target_negative_diff = L.Eltwise(self.net.inner_product2_target, self.net.inner_product2_negative,
name='target_negative_diff',
operation=P.Eltwise.SUM, # SUM
coeff=list([1,-1])) # target - negative
'''
#Context
context_sum_bottom = []
for i in range(0, self.number_of_neighbors):
context_sum_bottom.append(
getattr(self.net, 'inner_product_neighbor{0}'.format(i)))
coeff = 1.0 / self.number_of_neighbors
self.net.context_sum = L.Eltwise(
*context_sum_bottom,
name='context_sum',
operation=P.Eltwise.SUM, # SUM
coeff=list([coeff for i in range(self.number_of_neighbors)]))
#Target - Negative
self.net.target_negative_diff = L.Eltwise(
self.net.inner_product_target,
self.net.inner_product_negative,
name='target_negative_diff',
operation=P.Eltwise.SUM, # SUM
coeff=list([1, -1])) # target - negative
#Loss layer
self.net.loss = L.Python(self.net.context_sum,
self.net.target_negative_diff,
name='loss',
module='my_dot_product_layer',
layer='MyHingLossDotProductLayer')
def saveNetwork(self, path):
with open(path, 'w') as f:
f.write("force_backward:true\n" + str(self.net.to_proto()))
def train_tail(self, last_top):
n = NetSpec()
n.loss = L.SoftmaxWithLoss(bottom=[last_top, "label"])
return n.to_proto()
def __init__(self, number_of_neighbors, num_output=100, lr_mult):
self.number_of_neighbors = number_of_neighbors
#self.netP =
self.net = NetSpec()
self.shared_weight_counter = 0
self.num_output = num_output
def deploy_tail(self, last_top):
n = NetSpec()
n.score = L.Softmax(bottom=last_top)
return n.to_proto()
def gen_net(batch_size=512):
n=NetSpec();
n.data = L.DummyData(shape={"dim":[batch_size,3,96,96]})
n.select1 = L.DummyData(shape={"dim":[2]})
n.select2 = L.DummyData(shape={"dim":[2]})
n.label = L.DummyData(shape={"dim":[2]})
caffenet_stack(n.data, n)
n.first = L.BatchReindex(n.fc6, n.select1)
n.second = L.BatchReindex(n.fc6, n.select2)
n.fc6_concat=L.Concat(n.first, n.second);
n.fc7, n.bn7, n.relu7 = fc_relu(n.fc6_concat, 4096, batchnorm=True);
n.fc8, n.relu8 = fc_relu(n.relu7, 4096);
n.fc9 = L.InnerProduct(n.relu8, num_output=8,
weight_filler=dict(type='xavier'));
n.loss = L.SoftmaxWithLoss(n.fc9, n.label, loss_param=dict(normalization=P.Loss.NONE));
prot=n.to_proto()
prot.debug_info=True
return prot;
def get_phocnet(self,
word_image_lmdb_path,
phoc_lmdb_path,
phoc_size=604,
generate_deploy=False):
'''
Returns a NetSpec definition of the PHOCNet. The definition can then be transformed
into a protobuffer message by casting it into a str.
'''
n = NetSpec()
relu_in_place = True
# Data
if generate_deploy:
n.word_images = L.Input(shape=dict(dim=[1, 1, 100, 250]))
relu_in_place = False
else:
n.word_images, n.label = L.Data(batch_size=1,
backend=P.Data.LMDB,
source=word_image_lmdb_path,
prefetch=20,
transform_param=dict(
mean_value=255,
scale=-1. / 255,
),
ntop=2)
n.phocs, n.label_phocs = L.Data(batch_size=1,
backend=P.Data.LMDB,
source=phoc_lmdb_path,
prefetch=20,
ntop=2)
# Conv Part
n.conv1_1, n.relu1_1 = self.conv_relu(n.word_images,
nout=64,
relu_in_place=relu_in_place)
n.conv1_2, n.relu1_2 = self.conv_relu(n.relu1_1,
nout=64,
relu_in_place=relu_in_place)
n.pool1 = L.Pooling(n.relu1_2,
pooling_param=dict(pool=P.Pooling.MAX,
kernel_size=2,
stride=2))
n.conv2_1, n.relu2_1 = self.conv_relu(n.pool1,
nout=128,
relu_in_place=relu_in_place)
n.conv2_2, n.relu2_2 = self.conv_relu(n.relu2_1,
nout=128,
relu_in_place=relu_in_place)
n.pool2 = L.Pooling(n.relu2_2,
pooling_param=dict(pool=P.Pooling.MAX,
kernel_size=2,
stride=2))
n.conv3_1, n.relu3_1 = self.conv_relu(n.pool2,
nout=256,
relu_in_place=relu_in_place)
n.conv3_2, n.relu3_2 = self.conv_relu(n.relu3_1,
nout=256,
relu_in_place=relu_in_place)
n.conv3_3, n.relu3_3 = self.conv_relu(n.relu3_2,
nout=256,
relu_in_place=relu_in_place)
n.conv3_4, n.relu3_4 = self.conv_relu(n.relu3_3,
nout=256,
relu_in_place=relu_in_place)
n.conv3_5, n.relu3_5 = self.conv_relu(n.relu3_4,
nout=256,
relu_in_place=relu_in_place)
n.conv3_6, n.relu3_6 = self.conv_relu(n.relu3_5,
nout=256,
relu_in_place=relu_in_place)
n.conv4_1, n.relu4_1 = self.conv_relu(n.relu3_6,
nout=512,
relu_in_place=relu_in_place)
n.conv4_2, n.relu4_2 = self.conv_relu(n.relu4_1,
nout=512,
relu_in_place=relu_in_place)
n.conv4_3, n.relu4_3 = self.conv_relu(n.relu4_2,
nout=512,
relu_in_place=relu_in_place)
# FC Part
n.spp5 = L.SPP(n.relu4_3,
spp_param=dict(pool=P.SPP.MAX,
pyramid_height=3,
engine=self.spp_engine))
n.fc6, n.relu6, n.drop6 = self.fc_relu(bottom=n.spp5,
layer_size=4096,
dropout_ratio=0.5,
relu_in_place=relu_in_place)
n.fc7, n.relu7, n.drop7 = self.fc_relu(bottom=n.drop6,
layer_size=4096,
dropout_ratio=0.5,
relu_in_place=relu_in_place)
n.fc8 = L.InnerProduct(n.drop7,
num_output=phoc_size,
weight_filler=dict(type=self.initialization),
bias_filler=dict(type='constant'))
n.sigmoid = L.Sigmoid(n.fc8, include=dict(phase=self.phase_test))
# output part
if not generate_deploy:
n.silence = L.Silence(n.sigmoid,
ntop=0,
include=dict(phase=self.phase_test))
n.loss = L.SigmoidCrossEntropyLoss(n.fc8, n.phocs)
return n.to_proto()
def main():
from argparse import ArgumentParser
from os import path
import numpy as np
parser = ArgumentParser()
parser.add_argument('prototxt')
parser.add_argument('output_caffemodel')
parser.add_argument(
'-l',
'--load',
help=
'Load a pretrained model and rescale it [bias and type are not supported]'
)
parser.add_argument('-d',
'--data',
default=None,
help='Image list to use [default prototxt data]')
parser.add_argument('-b', '--bias', type=float, default=0.1, help='Bias')
parser.add_argument(
'-t',
'--type',
default='elwise',
help=
'Type: elwise, pca, zca, kmeans, rand (random input patches). Add fast_ to speed up the initialization, but you might lose in precision.'
)
parser.add_argument(
'--zero_from',
default=None,
help='Zero weights starting from this layer and reinitialize')
parser.add_argument('-z',
action='store_true',
help='Zero all weights and reinitialize')
parser.add_argument(
'--post_zero_from',
default=None,
help=
'AFTER everything else, zero weights starting from this layer (they will NOT be reinitialized)'
)
parser.add_argument('-cs', action='store_true', help='Correct for scaling')
parser.add_argument('-q', action='store_true', help='Quiet execution')
parser.add_argument('-s',
type=float,
default=1.0,
help='Scale the input [only custom data "-d"]')
parser.add_argument('-bs',
type=int,
default=16,
help='Batch size [only custom data "-d"]')
parser.add_argument('-nit',
type=int,
default=10,
help='Number of iterations')
parser.add_argument(
'--mem-limit',
type=int,
default=500,
help='How much memory should we use for the data buffer (MB)?')
parser.add_argument('--gpu',
type=int,
default=0,
help='What gpu to run it on?')
args = parser.parse_args()
if args.q:
from os import environ
environ['GLOG_minloglevel'] = '2'
import caffe, load
from caffe import NetSpec, layers as L
caffe.set_mode_gpu()
if args.gpu is not None:
caffe.set_device(args.gpu)
if args.data is not None:
model = load.ProtoDesc(args.prototxt)
net = NetSpec()
fl = getFileList(args.data)
if len(fl) == 0:
print("Unknown data type for '%s'" % args.data)
exit(1)
from tempfile import NamedTemporaryFile
f = NamedTemporaryFile('w')
f.write('\n'.join([path.abspath(i) + ' 0' for i in fl]))
f.flush()
net.data, net.label = L.ImageData(source=f.name,
batch_size=args.bs,
new_width=model.input_dim[-1],
new_height=model.input_dim[-1],
transform_param=dict(
mean_value=[104, 117, 123],
scale=args.s),
ntop=2)
net.out = model(data=net.data, label=net.label)
n = netFromString('force_backward:true\n' + str(net.to_proto()),
caffe.TRAIN)
else:
n = caffe.Net(args.prototxt, caffe.TRAIN)
if args.load is not None:
n.copy_from(args.load)
# Rescale existing layers?
#if args.fix:
#magicFix(n, args.nit)
if args.z or args.zero_from:
zeroLayers(n, start=args.zero_from)
if any([
np.abs(l.blobs[0].data).sum() < 1e-10 for l in n.layers
if len(l.blobs) > 0
]):
print([
m for l, m in zip(n.layers, n._layer_names)
if len(l.blobs) > 0 and np.abs(l.blobs[0].data).sum() < 1e-10
])
magicInitialize(n,
args.bias,
NIT=args.nit,
type=args.type,
max_data=args.mem_limit * 1024 * 1024 / 4)
else:
print("Network already initialized, skipping magic init")
if args.cs:
# A simply helper function that lets you figure out which layers are not
# homogeneous
#print( estimateHomogenety(n) )
print('Calibrating gradient ratio')
calibrateGradientRatio(n)
if args.post_zero_from:
zeroLayers(n, start=args.post_zero_from)
n.save(args.output_caffemodel)
def main():
from argparse import ArgumentParser
from os import path
parser = ArgumentParser()
parser.add_argument('prototxt')
parser.add_argument('-l', '--load', help='Load a caffemodel')
parser.add_argument('-d',
'--data',
default=None,
help='Image list to use [default prototxt data]')
#parser.add_argument('-q', action='store_true', help='Quiet execution')
parser.add_argument('-sm', action='store_true', help='Summary only')
parser.add_argument('-q', action='store_true', help='Quiet execution')
parser.add_argument('-a',
'--all',
action='store_true',
help='Show the statistic for all layers')
parser.add_argument('-nc', action='store_true', help='Do not use color')
parser.add_argument('-s',
type=float,
default=1.0,
help='Scale the input [only custom data "-d"]')
parser.add_argument('-bs',
type=int,
default=16,
help='Batch size [only custom data "-d"]')
parser.add_argument('-nit',
type=int,
default=10,
help='Number of iterations')
parser.add_argument('--gpu',
type=int,
default=0,
help='What gpu to run it on?')
args = parser.parse_args()
if args.q:
from os import environ
environ['GLOG_minloglevel'] = '2'
import caffe, load
from caffe import NetSpec, layers as L
caffe.set_mode_gpu()
if args.gpu is not None:
caffe.set_device(args.gpu)
if args.data is not None:
model = load.ProtoDesc(args.prototxt)
net = NetSpec()
fl = getFileList(args.data)
if len(fl) == 0:
print("Unknown data type for '%s'" % args.data)
exit(1)
from tempfile import NamedTemporaryFile
f = NamedTemporaryFile('w')
f.write('\n'.join([path.abspath(i) + ' 0' for i in fl]))
f.flush()
net.data, net.label = L.ImageData(source=f.name,
batch_size=args.bs,
new_width=model.input_dim[-1],
new_height=model.input_dim[-1],
transform_param=dict(
mean_value=[104, 117, 123],
scale=args.s),
ntop=2)
net.out = model(data=net.data, label=net.label)
n = netFromString('force_backward:true\n' + str(net.to_proto()),
caffe.TRAIN)
else:
n = caffe.Net(args.prototxt, caffe.TRAIN)
if args.load is not None:
n.copy_from(args.load)
cvar = printMeanStddev(n,
NIT=args.nit,
show_all=args.all,
show_color=not args.nc,
quiet=args.sm)
cv, gr = computeGradientRatio(n, NIT=args.nit)
print()
print(' Summary ')
print('-----------')
print()
print(
'layer name out cvar rate cvar rate mean'
)
for l in n._layer_names:
if l in cvar and l in cv and l in gr:
print('%-30s %10.2f %10.2f %10.2f' %
(l, cvar[l], cv[l], gr[l]))
def FireNet_generic(FireNet_module_func, choose_num_output_func, batch_size, pool_after, s):
print s
n = NetSpec()
FireNet_data_layer(n, batch_size) #add data layer to the net
layer_idx=1 #e.g. conv1, fire2, etc.
n.conv1 = L.Convolution(n.data, kernel_size=7, num_output=96, stride=2, weight_filler=dict(type='xavier'))
curr_bottom = 'conv1'
n.tops['relu_conv1'] = L.ReLU(n.tops[curr_bottom], in_place=True)
if curr_bottom in pool_after.keys():
curr_bottom = FireNet_pooling_layer(n, curr_bottom, pool_after[curr_bottom], layer_idx)
for layer_idx in xrange(2,10):
firenet_dict = choose_num_output_func(layer_idx-2, s)
print firenet_dict
curr_bottom = FireNet_module_func(n, curr_bottom, firenet_dict, layer_idx)
if curr_bottom in pool_after.keys():
curr_bottom = FireNet_pooling_layer(n, curr_bottom, pool_after[curr_bottom], layer_idx)
n.tops['drop'+str(layer_idx)] = L.Dropout(n.tops[curr_bottom], dropout_ratio=0.5, in_place=True)
n.tops['conv_final'] = L.Convolution(n.tops[curr_bottom], kernel_size=1, num_output=1000, weight_filler=dict(type='gaussian', std=0.01, mean=0.0))
n.tops['relu_conv_final'] = L.ReLU(n.tops['conv_final'], in_place=True)
n.tops['pool_final'] = L.Pooling(n.tops['conv_final'], global_pooling=1, pool=P.Pooling.AVE)
if phase == 'trainval':
n.loss = L.SoftmaxWithLoss(n.tops['pool_final'], n.label, include=dict(phase=caffe_pb2.TRAIN))
n.accuracy = L.Accuracy(n.tops['pool_final'], n.label, include=dict(phase=caffe_pb2.TEST))
n.accuracy_top5 = L.Accuracy(n.tops['pool_final'], n.label, include=dict(phase=caffe_pb2.TEST), top_k=5)
return n.to_proto()
def FireNet(batch_size, pool_after, s, c1):
print s
n = NetSpec()
FireNet_data_layer(n, batch_size) #add data layer to the net
layer_idx=1 #e.g. conv1, fire2, etc.
n.conv1 = L.Convolution(n.data, kernel_size=c1['dim'], num_output=c1['nfilt'], stride=2, weight_filler=dict(type='xavier'))
curr_bottom = 'conv1'
n.tops['relu_conv1'] = L.ReLU(n.tops[curr_bottom], in_place=True)
#if curr_bottom in pool_after.keys():
# curr_bottom = FireNet_pooling_layer(n, curr_bottom, pool_after[curr_bottom], layer_idx)
if layer_idx in pool_after:
n.tops['pool1'] = L.Pooling(n.tops[curr_bottom], kernel_size=3, stride=2, pool=P.Pooling.MAX)
curr_bottom = 'pool1'
for layer_idx in xrange(2, s['n_layers']+2):
firenet_dict = choose_num_output(layer_idx-2, s)
print firenet_dict
curr_bottom = FireNet_module(n, curr_bottom, firenet_dict, layer_idx)
if layer_idx in pool_after:
next_bottom = 'pool%d' %layer_idx
n.tops[next_bottom] = L.Pooling(n.tops[curr_bottom], kernel_size=3, stride=2, pool=P.Pooling.MAX)
curr_bottom = next_bottom
n.tops['drop'+str(layer_idx)] = L.Dropout(n.tops[curr_bottom], dropout_ratio=0.5, in_place=True)
#optional pre_conv_final (w/ appropriate CEratio)
#n.pre_conv_final = L.Convolution(n.tops[curr_bottom], kernel_size=1, num_output=int(1000*s['CEratio']), stride=1, weight_filler=dict(type='xavier'))
#n.tops['relu_pre_conv_final'] = L.ReLU(n.tops['pre_conv_final'], in_place=True)
#curr_bottom='pre_conv_final'
n.tops['conv_final'] = L.Convolution(n.tops[curr_bottom], kernel_size=1, num_output=1000, weight_filler=dict(type='gaussian', std=0.01, mean=0.0))
n.tops['relu_conv_final'] = L.ReLU(n.tops['conv_final'], in_place=True)
n.tops['pool_final'] = L.Pooling(n.tops['conv_final'], global_pooling=1, pool=P.Pooling.AVE)
if phase == 'trainval':
n.loss = L.SoftmaxWithLoss(n.tops['pool_final'], n.label, include=dict(phase=caffe_pb2.TRAIN))
n.accuracy = L.Accuracy(n.tops['pool_final'], n.label, include=dict(phase=caffe_pb2.TEST))
n.accuracy_top5 = L.Accuracy(n.tops['pool_final'], n.label, include=dict(phase=caffe_pb2.TEST), top_k=5)
return n.to_proto()
class Network:
def __init__(self, number_of_neighbors=6, inner_product_output=100, weight_lr_mult=1, weight_decay_mult=1, b_lr_mult=2, b_decay_mult=0):
self.number_of_neighbors = number_of_neighbors
#self.netP =
self.net = NetSpec()
self.shared_weight_counter = 0
self.num_output = inner_product_output
self.weight_lr_mult = weight_lr_mult
self.weight_decay_mult = weight_decay_mult
self.b_lr_mult = b_lr_mult
self.b_decay_mult = b_decay_mult
def getInnerProduct(self, input_name, output_name, ID, num_output=None):
#TODO What should be the output of this layer?
if num_output is None:
num_output = self.num_output
return L.InnerProduct(getattr(self.net, input_name),
name=output_name,
weight_filler=dict(type='xavier'),
bias_filler=dict(type='xavier', value=0.2),
num_output=num_output,
#in_place=True,
param=list([dict(name="embed_w{0}".format(ID), lr_mult=self.weight_lr_mult, decay_mult=self.weight_decay_mult),
dict(name="embed_b{0}".format(ID), lr_mult=self.b_lr_mult, decay_mult=self.b_decay_mult)])
)
def createEmbeddingNetwork(self, database_list_path='.', batch_size=20, phase=0):
dataset_path = database_list_path
dataLayer = L.HDF5Data(name='dataLayer',
source=dataset_path,
batch_size=batch_size,
ntop=2+self.number_of_neighbors,
include=list([dict(phase=phase)]))# tops-> target, [neighbors], negative
#data -> [target, neighbor1, neighbor2, ..., neighbork, negative]
self.net.target = dataLayer[0]
self.net.negative = dataLayer[-1]
for l in range(1, self.number_of_neighbors+1):
setattr(self.net, 'neighbor{0}'.format(l-1), dataLayer[l])
#First layer of inner product
self.net.inner_product_target = self.getInnerProduct('target', 'inner_product_target', 1)
self.net.inner_product_negative = self.getInnerProduct('negative', 'inner_product_negative', 1)
for i in range(0, self.number_of_neighbors):
layer = self.getInnerProduct('neighbor{0}'.format(i), 'inner_product_neighbor{0}'.format(i), 1)
setattr(self.net, 'inner_product_neighbor{0}'.format(i), layer)
#ReLU
self.net.relu_target = L.ReLU(self.net.inner_product_target, name='relu_target', in_place=True)
self.net.relu_negative = L.ReLU(self.net.inner_product_negative, name='relu_negative', in_place=True)
for i in range(0, self.number_of_neighbors):
layer = L.ReLU(getattr(self.net, 'inner_product_neighbor{0}'.format(i)),
name='relu_neighbor{0}'.format(i),
in_place=True)
setattr(self.net, 'relu_neighbor{0}'.format(i), layer)
#Second layer of inner product
#self.net.inner_product2_target = self.getInnerProduct('inner_product_target', 'inner_product2_target', 2)
#self.net.inner_product2_negative = self.getInnerProduct('inner_product_negative', 'inner_product2_negative', 2)
#for i in range(0, self.number_of_neighbors):
# layer = self.getInnerProduct('inner_product_neighbor{0}'.format(i),
# 'inner_product2_neighbor{0}'.format(i), 2)
# setattr(self.net, 'inner_product2_neighbor{0}'.format(i), layer)
#Context
'''
context_sum_bottom = []
for i in range(0, self.number_of_neighbors):
context_sum_bottom.append(getattr(self.net, 'inner_product2_neighbor{0}'.format(i)))
coeff = 1.0/self.number_of_neighbors
self.net.context_sum = L.Eltwise(*context_sum_bottom,
name='context_sum',
operation=P.Eltwise.SUM, # 1 -> SUM
coeff=list([coeff for i in range(self.number_of_neighbors)]))
#Target - Negative
self.net.target_negative_diff = L.Eltwise(self.net.inner_product2_target, self.net.inner_product2_negative,
name='target_negative_diff',
operation=P.Eltwise.SUM, # SUM
coeff=list([1,-1])) # target - negative
'''
#Context
context_sum_bottom = []
for i in range(0, self.number_of_neighbors):
context_sum_bottom.append(getattr(self.net, 'inner_product_neighbor{0}'.format(i)))
coeff = 1.0/self.number_of_neighbors
self.net.context_sum = L.Eltwise(*context_sum_bottom,
name='context_sum',
operation=P.Eltwise.SUM, # SUM
coeff=list([coeff for i in range(self.number_of_neighbors)]))
#Target - Negative
self.net.target_negative_diff = L.Eltwise(self.net.inner_product_target, self.net.inner_product_negative,
name='target_negative_diff',
operation=P.Eltwise.SUM, # SUM
coeff=list([1,-1])) # target - negative
#Loss layer
self.net.loss = L.Python(self.net.context_sum, self.net.target_negative_diff,
name='loss',
module='my_dot_product_layer',
layer='MyHingLossDotProductLayer')
def saveNetwork(self, model_prototxt_path):
with open(model_prototxt_path, 'w') as f:
f.write("force_backward:true\n"+str(self.net.to_proto()))
def __init__(self, number_of_neighbors, num_output=100, lr_mult): self.number_of_neighbors = number_of_neighbors #self.netP = self.net = NetSpec() self.shared_weight_counter = 0 self.num_output = num_output
def test_tail(self, last_top):
n = NetSpec()
n.accuracy = L.Accuracy(bottom=[last_top, "label"],
include=dict(phase=caffe.TEST))
return n.to_proto()
def main():
from argparse import ArgumentParser
from os import path
import numpy as np
parser = ArgumentParser()
parser.add_argument('prototxt')
parser.add_argument('output_caffemodel')
parser.add_argument('-l', '--load', help='Load a pretrained model and rescale it [bias and type are not supported]')
parser.add_argument('-d', '--data', default=None, help='Image list to use [default prototxt data]')
parser.add_argument('-b', '--bias', type=float, default=0.1, help='Bias')
parser.add_argument('-t', '--type', default='elwise', help='Type: elwise, pca, zca, kmeans, rand (random input patches). Add fast_ to speed up the initialization, but you might lose in precision.')
parser.add_argument('-z', action='store_true', help='Zero all weights and reinitialize')
parser.add_argument('-cs', action='store_true', help='Correct for scaling')
parser.add_argument('-q', action='store_true', help='Quiet execution')
parser.add_argument('-s', type=float, default=1.0, help='Scale the input [only custom data "-d"]')
parser.add_argument('-bs', type=int, default=16, help='Batch size [only custom data "-d"]')
parser.add_argument('-nit', type=int, default=10, help='Number of iterations')
parser.add_argument('--mem-limit', type=int, default=500, help='How much memory should we use for the data buffer (MB)?')
parser.add_argument('--gpu', type=int, default=0, help='What gpu to run it on?')
args = parser.parse_args()
if args.q:
from os import environ
environ['GLOG_minloglevel'] = '2'
import caffe, load
from caffe import NetSpec, layers as L
caffe.set_mode_gpu()
if args.gpu is not None:
caffe.set_device(args.gpu)
if args.data is not None:
model = load.ProtoDesc(args.prototxt)
net = NetSpec()
fl = getFileList(args.data)
if len(fl) == 0:
print("Unknown data type for '%s'"%args.data)
exit(1)
from tempfile import NamedTemporaryFile
f = NamedTemporaryFile('w')
f.write('\n'.join([path.abspath(i)+' 0' for i in fl]))
f.flush()
net.data, net.label = L.ImageData(source=f.name, batch_size=args.bs, new_width=model.input_dim[-1], new_height=model.input_dim[-1], transform_param=dict(mean_value=[104,117,123], scale=args.s),ntop=2)
net.out = model(data=net.data, label=net.label)
n = netFromString('force_backward:true\n'+str(net.to_proto()), caffe.TRAIN )
else:
n = caffe.Net(args.prototxt, caffe.TRAIN)
if args.load is not None:
n.copy_from(args.load)
# Rescale existing layers?
#if args.fix:
#magicFix(n, args.nit)
if args.z:
# Zero out all layers
for l in n.layers:
for b in l.blobs:
b.data[...] = 0
if any([np.abs(l.blobs[0].data).sum() < 1e-10 for l in n.layers if len(l.blobs) > 0]):
print( [m for l,m in zip(n.layers, n._layer_names) if len(l.blobs) > 0 and np.abs(l.blobs[0].data).sum() < 1e-10] )
magicInitialize(n, args.bias, NIT=args.nit, type=args.type, max_data=args.mem_limit*1024*1024/4)
else:
print( "Network already initialized, skipping magic init" )
if args.cs:
# A simply helper function that lets you figure out which layers are not
# homogeneous
#print( estimateHomogenety(n) )
calibrateGradientRatio(n)
n.save(args.output_caffemodel)
class Network:
def __init__(self,
number_of_neighbors=6,
inner_product_output=100,
weight_lr_mult=1,
weight_decay_mult=1,
b_lr_mult=2,
b_decay_mult=0,
inner_product_output_l1=1000):
self.number_of_neighbors = number_of_neighbors
#self.netP =
self.net = NetSpec()
self.shared_weight_counter = 0
self.num_output = inner_product_output
self.weight_lr_mult = weight_lr_mult
self.weight_decay_mult = weight_decay_mult
self.b_lr_mult = b_lr_mult
self.b_decay_mult = b_decay_mult
self.inner_product_output_l1 = inner_product_output_l1
def getInnerProduct(self, input_name, output_name, ID, num_output=None):
#TODO What should be the output of this layer?
if num_output is None:
num_output = self.num_output
return L.InnerProduct(
getattr(self.net, input_name),
name=output_name,
weight_filler=dict(type='xavier'),
bias_filler=dict(type='xavier', value=0.2),
num_output=num_output,
#in_place=True,
param=list([
dict(name="embed_w{0}".format(ID),
lr_mult=self.weight_lr_mult,
decay_mult=self.weight_decay_mult),
dict(name="embed_b{0}".format(ID),
lr_mult=self.b_lr_mult,
decay_mult=self.b_decay_mult)
]))
def createEmbeddingNetwork(self,
database_list_path='.',
batch_size=20,
phase=0):
dataset_path = database_list_path
dataLayer = L.HDF5Data(name='dataLayer',
source=dataset_path,
batch_size=batch_size,
ntop=3 + self.number_of_neighbors,
include=list([
dict(phase=phase)
])) # tops-> target, [neighbors], negative
#data -> [target, neighbor1, neighbor2, ..., neighbork, negative]
self.net.target = dataLayer[0]
self.net.negative = dataLayer[-2]
self.net.data_weights = dataLayer[-1]
for l in range(1, self.number_of_neighbors + 1):
setattr(self.net, 'neighbor{0}'.format(l - 1), dataLayer[l])
#First layer of inner product
layer_output = self.inner_product_output_l1
self.net.inner_product_target_1 = self.getInnerProduct(
'target', 'inner_product_target_1', 2, num_output=layer_output)
self.net.inner_product_negative_1 = self.getInnerProduct(
'negative', 'inner_product_negative_1', 2, num_output=layer_output)
for i in range(0, self.number_of_neighbors):
layer = self.getInnerProduct(
'neighbor{0}'.format(i),
'inner_product_neighbor{0}_1'.format(i),
2,
num_output=layer_output)
setattr(self.net, 'inner_product_neighbor{0}_1'.format(i), layer)
#Relu on top of the fisrt inner product
self.net.relu_target_1 = L.ReLU(self.net.inner_product_target_1,
name='relu_target_1',
in_place=True)
self.net.relu_negative_1 = L.ReLU(self.net.inner_product_negative_1,
name='relu_negative_1',
in_place=True)
for i in range(0, self.number_of_neighbors):
layer = L.ReLU(getattr(self.net,
'inner_product_neighbor{0}_1'.format(i)),
name='relu_neighbor{0}_1'.format(i),
in_place=True)
setattr(self.net, 'relu_neighbor{0}_1'.format(i), layer)
#Dropout1
#self.net.dropout_target_1 = L.Dropout(self.net.relu_target_1, name='dropout_target_1', in_place=True)
#self.net.dropout_negative_1 = L.Dropout(self.net.relu_negative_1, name='dropout_negative_1', in_place=True)
#Second layer of inner product
self.net.inner_product_target = self.getInnerProduct(
'inner_product_target_1', 'inner_product_target', 1)
self.net.inner_product_negative = self.getInnerProduct(
'inner_product_negative_1', 'inner_product_negative', 1)
for i in range(0, self.number_of_neighbors):
layer = self.getInnerProduct(
'inner_product_neighbor{0}_1'.format(i),
'inner_product_neighbor{0}'.format(i), 1)
setattr(self.net, 'inner_product_neighbor{0}'.format(i), layer)
#ReLU2
self.net.relu_target = L.ReLU(self.net.inner_product_target,
name='relu_target',
in_place=True)
self.net.relu_negative = L.ReLU(self.net.inner_product_negative,
name='relu_negative',
in_place=True)
for i in range(0, self.number_of_neighbors):
layer = L.ReLU(getattr(self.net,
'inner_product_neighbor{0}'.format(i)),
name='relu_neighbor{0}'.format(i),
in_place=True)
setattr(self.net, 'relu_neighbor{0}'.format(i), layer)
#self.net.normalize_target = L.NormalizeLayer(self.net.inner_product_target, name='normalize_target', in_place=True)
#self.net.normalize_negative = L.NormalizeLayer(self.net.inner_product_negative, name='normalize_negative', in_place=True)
#for i in range(0, self.number_of_neighbors):
# layer = L.NormalizeLayer(getattr(self.net, 'inner_product_neighbor{0}'.format(i)),
# name='normalize_neighbor{0}'.format(i),
# in_place=True)
# setattr(self.net, 'normalize_neighbor{0}'.format(i), layer)
#Second layer of inner product
#self.net.inner_product2_target = self.getInnerProduct('inner_product_target', 'inner_product2_target', 2)
#self.net.inner_product2_negative = self.getInnerProduct('inner_product_negative', 'inner_product2_negative', 2)
#for i in range(0, self.number_of_neighbors):
# layer = self.getInnerProduct('inner_product_neighbor{0}'.format(i),
# 'inner_product2_neighbor{0}'.format(i), 2)
# setattr(self.net, 'inner_product2_neighbor{0}'.format(i), layer)
#Context
'''
context_sum_bottom = []
for i in range(0, self.number_of_neighbors):
context_sum_bottom.append(getattr(self.net, 'inner_product2_neighbor{0}'.format(i)))
coeff = 1.0/self.number_of_neighbors
self.net.context_sum = L.Eltwise(*context_sum_bottom,
name='context_sum',
operation=P.Eltwise.SUM, # 1 -> SUM
coeff=list([coeff for i in range(self.number_of_neighbors)]))
#Target - Negative
self.net.target_negative_diff = L.Eltwise(self.net.inner_product2_target, self.net.inner_product2_negative,
name='target_negative_diff',
operation=P.Eltwise.SUM, # SUM
coeff=list([1,-1])) # target - negative
'''
#Context
context_sum_bottom = []
for i in range(0, self.number_of_neighbors):
context_sum_bottom.append(
getattr(self.net, 'inner_product_neighbor{0}'.format(i)))
coeff = 1.0 / self.number_of_neighbors
self.net.context_sum = L.Eltwise(
*context_sum_bottom,
name='context_sum',
operation=P.Eltwise.SUM, # SUM
coeff=list([coeff for i in range(self.number_of_neighbors)]))
#self.net.
#Target - Negative
self.net.target_negative_diff = L.Eltwise(
self.net.inner_product_target,
self.net.inner_product_negative,
name='target_negative_diff',
operation=P.Eltwise.SUM, # SUM
coeff=list([1, -1])) # target - negative
self.net.target_negative_diff_weighted = L.Eltwise(
self.net.target_negative_diff,
self.net.data_weights,
name='target_negative_diff_weighted',
operation=P.Eltwise.PROD)
self.net.weights = L.Eltwise(self.net.inner_product_target,
self.net.inner_product_negative,
name='similarity',
operation=P.Eltwise.PROD)
#Loss layer
self.net.silence_data = L.Silence(self.net.weights, ntop=0)
self.net.loss = L.Python(self.net.context_sum,
self.net.target_negative_diff_weighted,
name='loss',
module='my_dot_product_layer',
layer='MyHingLossDotProductLayer',
loss_weight=1)
def saveNetwork(self, model_prototxt_path):
with open(model_prototxt_path, 'w') as f:
f.write("force_backward:true\n" + str(self.net.to_proto()))
def get_train_data_layer(batch_size):
#important: conf.train_lmdb
n = NetSpec()
n.data, n.label = L.Data(batch_size=batch_size, backend=P.Data.LMDB, source=conf.train_lmdb, include=dict(phase=caffe_pb2.TRAIN),
transform_param=dict(crop_size=227, mean_value=[104, 117, 123]), ntop=2)
return n.to_proto()
def lenet(lmdbData, lmdbLabel, batch_size):
n = NetSpec()
n.data = L.Data(batch_size=batch_size, backend=P.Data.LMDB, source=lmdbData,
transform_param=dict(scale=1./255), ntop=1)
n.label = L.Data(batch_size=batch_size, backend=P.Data.LMDB, source=lmdbLabel,
transform_param=dict(scale=1./255), ntop=1)
n.conv1 = L.Convolution(n.data, kernel_size=4, num_output=200, weight_filler=dict(type='xavier'))
n.pool1 = L.Pooling(n.conv1, kernel_size=2, stride=2, pool=P.Pooling.MAX)
n.conv2 = L.Convolution(n.pool1, kernel_size=3, num_output=50, weight_filler=dict(type='xavier'))
n.pool2 = L.Pooling(n.conv2, kernel_size=2, stride=1, pool=P.Pooling.MAX)
n.fc1 = L.InnerProduct(n.pool2, num_output=200, weight_filler=dict(type='xavier'))
n.relu1 = L.ReLU(n.fc1, in_place=True)
n.score = L.InnerProduct(n.relu1, num_output=1200, weight_filler=dict(type='xavier'))
n.loss = L.Python(n.score, n.label, module='pyloss', layer='EuclideanLossLayer')
return n.to_proto()
class Network:
def __init__(self, number_of_neighbors=6, inner_product_output=100, weight_lr_mult=1, weight_decay_mult=1, b_lr_mult=2, b_decay_mult=0):
self.number_of_neighbors = number_of_neighbors
#self.netP =
self.net = NetSpec()
self.shared_weight_counter = 0
self.num_output = inner_product_output
self.weight_lr_mult = weight_lr_mult
self.weight_decay_mult = weight_decay_mult
self.b_lr_mult = b_lr_mult
self.b_decay_mult = b_decay_mult
def getInnerProduct(self, input_name, output_name, ID, num_output=None):
#TODO What should be the output of this layer?
if num_output is None:
num_output = self.num_output
return L.InnerProduct(getattr(self.net, input_name),
name=output_name,
weight_filler=dict(type='xavier'),
bias_filler=dict(type='xavier', value=0.2),
num_output=num_output,
#in_place=True,
param=list([dict(name="embed_w{0}".format(ID), lr_mult=self.weight_lr_mult, decay_mult=self.weight_decay_mult),
dict(name="embed_b{0}".format(ID), lr_mult=self.b_lr_mult, decay_mult=self.b_decay_mult)])
)
def l2normed(self,vec, dim):
#Returns L2-normalized instances of vec; i.e., for each instance x in vec,
#computes x / ((x ** 2).sum() ** 0.5). Assumes vec has shape N x dim."""
denom = L.Reduction(vec, axis=1, operation=P.Reduction.SUMSQ)
denom = L.Power(denom, power=(-0.5), shift=1e-12)
denom = L.Reshape(denom, num_axes=0, axis=-1, shape=dict(dim=[1]))
denom = L.Tile(denom, axis=1, tiles=dim)
return L.Eltwise(vec, denom, operation=P.Eltwise.PROD)
def createEmbeddingNetwork(self, database_list_path='.', batch_size=20, phase=0):
dataset_path = database_list_path
dataLayer = L.HDF5Data(name='dataLayer',
source=dataset_path,
batch_size=batch_size,
ntop=2+self.number_of_neighbors,
include=list([dict(phase=phase)]))# tops-> target, [neighbors], negative
#data -> [target, neighbor1, neighbor2, ..., neighbork, negative]
self.net.target = dataLayer[0]
self.net.negative = dataLayer[-1]
for l in range(1, self.number_of_neighbors+1):
setattr(self.net, 'neighbor{0}'.format(l-1), dataLayer[l])
#First layer of inner product
self.net.inner_product_target = self.getInnerProduct('target', 'inner_product_target', 1)
self.net.inner_product_negative = self.getInnerProduct('negative', 'inner_product_negative', 1)
for i in range(0, self.number_of_neighbors):
layer = self.getInnerProduct('neighbor{0}'.format(i), 'inner_product_neighbor{0}'.format(i), 1)
setattr(self.net, 'inner_product_neighbor{0}'.format(i), layer)
#ReLU
self.net.relu_target = L.ReLU(self.net.inner_product_target, name='relu_target', in_place=True)
self.net.relu_negative = L.ReLU(self.net.inner_product_negative, name='relu_negative', in_place=True)
for i in range(0, self.number_of_neighbors):
layer = L.ReLU(getattr(self.net, 'inner_product_neighbor{0}'.format(i)),
name='relu_neighbor{0}'.format(i),
in_place=True)
setattr(self.net, 'relu_neighbor{0}'.format(i), layer)
#self.net.normalize_target = L.NormalizeLayer(self.net.inner_product_target, name='normalize_target', in_place=True)
#self.net.normalize_negative = L.NormalizeLayer(self.net.inner_product_negative, name='normalize_negative', in_place=True)
#for i in range(0, self.number_of_neighbors):
# layer = L.NormalizeLayer(getattr(self.net, 'inner_product_neighbor{0}'.format(i)),
# name='normalize_neighbor{0}'.format(i),
# in_place=True)
# setattr(self.net, 'normalize_neighbor{0}'.format(i), layer)
#Second layer of inner product
#self.net.inner_product2_target = self.getInnerProduct('inner_product_target', 'inner_product2_target', 2)
#self.net.inner_product2_negative = self.getInnerProduct('inner_product_negative', 'inner_product2_negative', 2)
#for i in range(0, self.number_of_neighbors):
# layer = self.getInnerProduct('inner_product_neighbor{0}'.format(i),
# 'inner_product2_neighbor{0}'.format(i), 2)
# setattr(self.net, 'inner_product2_neighbor{0}'.format(i), layer)
#Context
'''
context_sum_bottom = []
for i in range(0, self.number_of_neighbors):
context_sum_bottom.append(getattr(self.net, 'inner_product2_neighbor{0}'.format(i)))
coeff = 1.0/self.number_of_neighbors
self.net.context_sum = L.Eltwise(*context_sum_bottom,
name='context_sum',
operation=P.Eltwise.SUM, # 1 -> SUM
coeff=list([coeff for i in range(self.number_of_neighbors)]))
#Target - Negative
self.net.target_negative_diff = L.Eltwise(self.net.inner_product2_target, self.net.inner_product2_negative,
name='target_negative_diff',
operation=P.Eltwise.SUM, # SUM
coeff=list([1,-1])) # target - negative
'''
#Context
context_sum_bottom = []
for i in range(0, self.number_of_neighbors):
context_sum_bottom.append(getattr(self.net, 'inner_product_neighbor{0}'.format(i)))
coeff = 1.0/self.number_of_neighbors
self.net.context_sum = L.Eltwise(*context_sum_bottom,
name='context_sum',
operation=P.Eltwise.SUM, # SUM
coeff=list([coeff for i in range(self.number_of_neighbors)]))
#self.net.
#Next section: normalizing embedding
#Target - Negative
self.net.target_norm = self.l2normed(self.net.inner_product_target, self.num_output)
self.net.negative_norm = self.l2normed(self.net.inner_product_negative, self.num_output)
self.net.context_norm = self.l2normed(self.net.context_sum, self.num_output)
self.net.target_negative_diff = L.Eltwise(self.net.target_norm, self.net.negative_norm,
name='target_negative_diff',
operation=P.Eltwise.SUM, # SUM
coeff=list([1,-1])) # target - negative
#Loss layer
self.net.loss = L.Python(self.net.context_norm, self.net.target_negative_diff,
name='loss',
module='my_dot_product_layer',
layer='MyHingLossDotProductLayer')
#End of normalizing
#Target - Negative
# self.net.target_negative_diff = L.Eltwise(self.net.inner_product_target, self.net.inner_product_negative,
# name='target_negative_diff',
# operation=P.Eltwise.SUM, # SUM
# coeff=list([1,-1])) # target - negative
#
#
# #Loss layer
# self.net.loss = L.Python(self.net.context_sum, self.net.target_negative_diff,
# name='loss',
# module='my_dot_product_layer',
# layer='MyHingLossDotProductLayer')
def saveNetwork(self, model_prototxt_path):
with open(model_prototxt_path, 'w') as f:
f.write("force_backward:true\n"+str(self.net.to_proto()))
class Network:
def __init__(self, number_of_neighbors=6, inner_product_output=100, weight_lr_mult=1, weight_decay_mult=1, b_lr_mult=2, b_decay_mult=0, feature_type=[]):
self.number_of_neighbors = number_of_neighbors
#self.netP =
self.net = NetSpec()
self.shared_weight_counter = 0
self.num_output = inner_product_output
self.weight_lr_mult = weight_lr_mult
self.weight_decay_mult = weight_decay_mult
self.b_lr_mult = b_lr_mult
self.b_decay_mult = b_decay_mult
self.feature_type = [x.name for x in feature_type]
def getInnerProduct(self, input_name, output_name, ID, num_output=None):
#TODO What should be the output of this layer?
if num_output is None:
num_output = self.num_output
return L.InnerProduct(getattr(self.net, input_name),
name=output_name,
weight_filler=dict(type='xavier'),
bias_filler=dict(type='xavier', value=0.2),
num_output=num_output,
#in_place=True,
param=list([dict(name="embed_w{0}".format(ID), lr_mult=self.weight_lr_mult, decay_mult=self.weight_decay_mult),
dict(name="embed_b{0}".format(ID), lr_mult=self.b_lr_mult, decay_mult=self.b_decay_mult)])
)
def getInnerProduct_with_relu(self, input_name, output_name, ID, num_output=None):
#TODO What should be the output of this layer?
if num_output is None:
num_output = self.num_output
lay = L.InnerProduct(getattr(self.net, input_name),
name=output_name,
weight_filler=dict(type='xavier'),
bias_filler=dict(type='xavier', value=0.2),
num_output=num_output,
#in_place=True,
param=list([dict(name="embed_w{0}".format(ID), lr_mult=self.weight_lr_mult, decay_mult=self.weight_decay_mult),
dict(name="embed_b{0}".format(ID), lr_mult=self.b_lr_mult, decay_mult=self.b_decay_mult)])
)
setattr(self.net, output_name, lay)
return L.ReLU(lay, name=output_name.replace('inner_product', 'relu'), in_place=True)
def return_layer(self, feature_key, layer_key):
#layer = L.InnerProductLayer .....
setattr(self.net, 'inner_product_layer_{0}_{1}'.format(feature_key, layer_key), layer)
return
def createEmbeddingNetwork(self, database_list_path='.', batch_size=20, phase=0):
dataset_path = database_list_path
dataLayer = L.HDF5Data(name='dataLayer',
source=dataset_path,
batch_size=batch_size,
ntop=len(self.feature_type)*(2+self.number_of_neighbors),
include=list([dict(phase=phase)]))# tops-> target, [neighbors], negative
#data -> [target, neighbor1, neighbor2, ..., neighbork, negative]
feature_type = self.feature_type
for i in xrange(len(self.feature_type)):
start = (i - 1)*(len(dataLayer)/len(self.feature_type))
end = i*(len(dataLayer)/len(self.feature_type)) - 1
setattr(self.net,'{0}_target'.format(feature_type[i]), dataLayer[start])
setattr(self.net,'{0}_negative'.format(feature_type[i]), dataLayer[end])
for l in range(start+1, end):
setattr(self.net, '{0}_neighbor{1}'.format(feature_type[i], l- start - 1), dataLayer[l])
#First layer of inner product
layer = self.getInnerProduct_with_relu('{0}_target'.format(feature_type[i]), '{0}_inner_product_target_1'.format(feature_type[i]), '_{}_2'.format(self.feature_type[i]), num_output=1000)
setattr(self.net, '{0}_relu_target_1'.format(feature_type[i]), layer)
layer = self.getInnerProduct_with_relu('{0}_negative'.format(feature_type[i]), '{0}_inner_product_negative_1'.format(feature_type[i]), '_{}_2'.format(self.feature_type[i]), num_output=1000)
setattr(self.net, '{0}_relu_negative_1'.format(feature_type[i]), layer)
for j in range(self.number_of_neighbors):
layer = self.getInnerProduct_with_relu('{0}_neighbor{1}'.format(feature_type[i], j), '{0}_inner_product_neighbor{1}_1'.format(feature_type[i], j), '_{}_2'.format(self.feature_type[i]), num_output=1000)
setattr(self.net, '{0}_relu_neighbor{1}_1'.format(feature_type[i], j), layer)
#Relu on top of the fisrt inner product
concat_inputs = ['{0}_relu'.format(x) for x in feature_type]
self.net.inner_product_target_1 = L.Concat(*map(lambda x: getattr(self.net, x + '_target_1'),concat_inputs), name='inner_product_target_1', axis=1)
self.net.inner_product_negative_1 = L.Concat(*map(lambda x: getattr(self.net, x + '_negative_1'),concat_inputs), name='inner_product_negative_1', axis=1)
for i in range(self.number_of_neighbors):
layer = L.Concat(*map(lambda x: getattr(self.net, x + '_neighbor{0}_1'.format(i)),concat_inputs), name='inner_product_neighbor{0}_1'.format(i), axis=1)
setattr(self.net, 'inner_product_neighbor{0}_1'.format(i), layer)
#for i in xrange(len(self.feature_type)):
# self.inner_product_target1 =
#Second layer of inner product
self.net.inner_product_target = self.getInnerProduct_with_relu('inner_product_target_1', 'inner_product_target', 1)
self.net.inner_product_negative = self.getInnerProduct_with_relu('inner_product_negative_1', 'inner_product_negative', 1)
for i in range(0, self.number_of_neighbors):
layer = self.getInnerProduct_with_relu('inner_product_neighbor{0}_1'.format(i), 'inner_product_neighbor{0}'.format(i), 1)
setattr(self.net, 'relu_neighbor{0}'.format(i), layer)
#self.net.normalize_target = L.NormalizeLayer(self.net.inner_product_target, name='normalize_target', in_place=True)
#self.net.normalize_negative = L.NormalizeLayer(self.net.inner_product_negative, name='normalize_negative', in_place=True)
#for i in range(0, self.number_of_neighbors):
# layer = L.NormalizeLayer(getattr(self.net, 'inner_product_neighbor{0}'.format(i)),
# name='normalize_neighbor{0}'.format(i),
# in_place=True)
# setattr(self.net, 'normalize_neighbor{0}'.format(i), layer)
#Second layer of inner product
#self.net.inner_product2_target = self.getInnerProduct('inner_product_target', 'inner_product2_target', 2)
#self.net.inner_product2_negative = self.getInnerProduct('inner_product_negative', 'inner_product2_negative', 2)
#for i in range(0, self.number_of_neighbors):
# layer = self.getInnerProduct('inner_product_neighbor{0}'.format(i),
# 'inner_product2_neighbor{0}'.format(i), 2)
# setattr(self.net, 'inner_product2_neighbor{0}'.format(i), layer)
#Context
'''
context_sum_bottom = []
for i in range(0, self.number_of_neighbors):
context_sum_bottom.append(getattr(self.net, 'inner_product2_neighbor{0}'.format(i)))
coeff = 1.0/self.number_of_neighbors
self.net.context_sum = L.Eltwise(*context_sum_bottom,
name='context_sum',
operation=P.Eltwise.SUM, # 1 -> SUM
coeff=list([coeff for i in range(self.number_of_neighbors)]))
#Target - Negative
self.net.target_negative_diff = L.Eltwise(self.net.inner_product2_target, self.net.inner_product2_negative,
name='target_negative_diff',
operation=P.Eltwise.SUM, # SUM
coeff=list([1,-1])) # target - negative
'''
#Context
context_sum_bottom = []
for i in range(0, self.number_of_neighbors):
context_sum_bottom.append(getattr(self.net, 'inner_product_neighbor{0}'.format(i)))
coeff = 1.0/self.number_of_neighbors
self.net.context_sum = L.Eltwise(*context_sum_bottom,
name='context_sum',
operation=P.Eltwise.SUM, # SUM
coeff=list([coeff for i in range(self.number_of_neighbors)]))
#self.net.
#Target - Negative
self.net.target_negative_diff = L.Eltwise(self.net.inner_product_target, self.net.inner_product_negative,
name='target_negative_diff',
operation=P.Eltwise.SUM, # SUM
coeff=list([1,-1])) # target - negative
#Loss layer
self.net.loss = L.Python(self.net.context_sum, self.net.target_negative_diff,
name='loss',
module='my_dot_product_layer',
layer='MyHingLossDotProductLayer')
def saveNetwork(self, model_prototxt_path):
with open(model_prototxt_path, 'w') as f:
f.write("force_backward:true\n"+str(self.net.to_proto()))
def StickNet(batch_size, s):
inImgH = 224 #TODO: put inImg{H,W} into 's' if necessary.
inImgW = 224
round_to_nearest = 4
n = NetSpec()
FireNet_data_layer(n, batch_size) #add data layer to the net
curr_bottom='data'
#layer-to-layer counters
_totalStride = 1 #note that, using 1x1 conv, our (stride>1) is only in pooling layers.
_numPoolings = 1 #for indexing 'conv2_1', etc.
_ch=3
[activH, activW] = est_activ_size(inImgH, inImgW, _totalStride)
n_filt = choose_num_output(1, 1, _ch, activH, activW, s['mflop_per_img_target'], s['n_layers']) #only using this for conv1 to avoid oscillations.
n_filt = round_to(n_filt, round_to_nearest) #make divisible by 8
#FIXME: somehow account for num_output produced by conv1 when selecting number of filters for conv2. (else, conv2 goes way over budget on flops.)
# perhaps we need to find the number N such that N^2*activations = mflop_per_img_target?
idx_minor = 1
idx_major = 1
#this goes to (n_layers-1) ... then we do conv_final separately because it has a different weight init.
for layer_idx in xrange(1, s['n_layers']):
layer_str = '%d.%d' %(idx_major, idx_minor)
#select number of filters in this layer:
#[activH, activW] = est_activ_size(inImgH, inImgW, _totalStride)
#n_filt = choose_num_output(1, 1, _ch, activH, activW, s['mflop_per_img_target'], s['n_layers'])
#TODO: to avoid oscillations, perhaps just use choose_num_output for conv1,
# and then just double n_filt whenever we do stride=2.
#generate layer
ksize=1
stride=1
pad=0
curr_bottom = conv_relu_xavier(n, ksize, n_filt, layer_str, stride, pad, curr_bottom)
_ch = n_filt #for next layer
if layer_idx in s['pool_after'].keys():
pinfo = s['pool_after'][layer_idx]
#next_bottom = 'pool%d' %layer_idx
next_bottom = 'pool_' + layer_str
n.tops[next_bottom] = L.Pooling(n.tops[curr_bottom], kernel_size=pinfo['kernel_size'], stride=pinfo['stride'], pool=P.Pooling.MAX)
curr_bottom = next_bottom
_totalStride = _totalStride * pinfo['stride']
_numPoolings = _numPoolings + 1
n_filt = n_filt * pinfo['stride'] #to keep (most) layers at roughly the same complexity-per-layer
idx_major = idx_major + 1
idx_minor = 1
else:
idx_minor = idx_minor + 1
n.tops['drop'+str(layer_idx)] = L.Dropout(n.tops[curr_bottom], dropout_ratio=0.5, in_place=True)
n.tops['conv_final'] = L.Convolution(n.tops[curr_bottom], kernel_size=1, num_output=1000, weight_filler=dict(type='gaussian', std=0.01, mean=0.0))
n.tops['relu_conv_final'] = L.ReLU(n.tops['conv_final'], in_place=True)
n.tops['pool_final'] = L.Pooling(n.tops['conv_final'], global_pooling=1, pool=P.Pooling.AVE)
if phase == 'trainval':
n.loss = L.SoftmaxWithLoss(n.tops['pool_final'], n.label, include=dict(phase=caffe_pb2.TRAIN))
n.accuracy = L.Accuracy(n.tops['pool_final'], n.label, include=dict(phase=caffe_pb2.TEST))
n.accuracy_top5 = L.Accuracy(n.tops['pool_final'], n.label, include=dict(phase=caffe_pb2.TEST), top_k=5)
return n.to_proto()
