def test_caffe2_to_onnx(self):
caffe2_net = tempfile.NamedTemporaryFile()
caffe2_init_net = tempfile.NamedTemporaryFile()
output = tempfile.NamedTemporaryFile()
model = ModelHelper(name='caffe2-to-onnx-test')
brew.relu(model, ["X"], "Y")
caffe2_net.write(model.net.Proto().SerializeToString())
caffe2_net.flush()
init_model = ModelHelper(name='caffe2-to-onnx-init-test')
init_model.net.GivenTensorFill([], 'X', shape=[2, 2],
values=np.zeros((2, 2)).flatten().astype(float))
caffe2_init_net.write(init_model.net.Proto().SerializeToString())
caffe2_init_net.flush()
result = self._run_command(
caffe2_to_onnx, [
caffe2_net.name,
'--caffe2-init-net', caffe2_init_net.name,
'--output', output.name,
],
catch_exceptions=False,
)
onnx_model = ModelProto()
onnx_model.ParseFromString(output.read())
self.assertEqual(len(onnx_model.graph.node), 1)
self.assertEqual(onnx_model.graph.node[0].op_type, 'Relu')
self.assertEqual(len(onnx_model.graph.initializer), 1)
self.assertEqual(onnx_model.graph.initializer[0].name, onnx_model.graph.input[0].name)
def AddLeNetModel(model, data):
conv1 = brew.conv(model,
data,
'conv1',
dim_in=1,
dim_out=32,
kernel=3,
pad=1)
conv1 = brew.relu(model, conv1, conv1)
conv2 = brew.conv(model,
conv1,
'conv2',
dim_in=32,
dim_out=128,
kernel=3,
pad=1)
conv2 = brew.relu(model, conv2, conv2)
conv3 = brew.conv(model,
conv2,
'conv3',
dim_in=128,
dim_out=256,
kernel=3,
pad=1)
conv3 = brew.relu(model, conv3, conv3)
fc3 = brew.fc(model, conv3, 'fc3', dim_in=256 * 28 * 28, dim_out=512)
fc3 = brew.relu(model, fc3, fc3)
pred = brew.fc(model, fc3, 'pred', 512, 10)
softmax = brew.softmax(model, pred, 'softmax')
return softmax
def test_caffe2_to_onnx_value_info(self):
caffe2_net = tempfile.NamedTemporaryFile()
output = tempfile.NamedTemporaryFile()
model = ModelHelper(name='caffe2-to-onnx-test')
brew.relu(model, ["X"], "Y")
caffe2_net.write(model.net.Proto().SerializeToString())
caffe2_net.flush()
args = [caffe2_net.name, '--output', output.name]
self.assertRaisesRegexp(Exception,
'value info',
self._run_command, caffe2_to_onnx, args)
args.extend([
'--value-info',
json.dumps({
'X': (TensorProto.FLOAT, (2, 2)),
})])
result = self._run_command(caffe2_to_onnx, args)
onnx_model = ModelProto()
onnx_model.ParseFromString(output.read())
self.assertEqual(len(onnx_model.graph.node), 1)
self.assertEqual(onnx_model.graph.node[0].op_type, 'Relu')
self.assertEqual(len(onnx_model.graph.initializer), 0)
def add_layers(model, data):
# Image size: 32 x 32 x 3 -> 30 x 30 x 60
conv1 = brew.conv(model, data, 'conv1', dim_in=3, dim_out=64, kernel=3)
#add relu
conv1 = brew.relu(model, conv1, conv1)
# Image size: 30 x 30 -> 15 x 15
pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2)
# Image size: 15 x 15 -> 12 x 12
conv2 = brew.conv(model, pool1, 'conv2', dim_in=64, dim_out=128, kernel=3)
#add relu
conv2 = brew.relu(model, conv2, conv2)
# Image size: 12 x 12 -> 6 x 6
pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2)
# 32 * 6 * 6 stands for dim_out from previous layer multiplied by the image size
# Hidden layer
fc3 = brew.fc(model, pool2, 'fc3', dim_in=128 * 6 * 6, dim_out=512)
#add relu
fc3 = brew.relu(model, fc3, fc3)
#CO = model.param_init_net.ConstantFill([], "CO", shape=[1], value=0.18)
#fc3 = brew.prelu(model, fc3, fc3)
#output layer
pred = brew.fc(model, fc3, 'pred', 512, 10)
#add softmax
probs = brew.softmax(model, pred, 'probs')
return probs
def test_caffe2_to_onnx_value_info(self):
caffe2_net = tempfile.NamedTemporaryFile()
output = tempfile.NamedTemporaryFile()
model = ModelHelper(name='caffe2-to-onnx-test')
brew.relu(model, ["X"], "Y")
caffe2_net.write(model.net.Proto().SerializeToString())
caffe2_net.flush()
args = [caffe2_net.name, '--output', output.name]
self.assertRaisesRegexp(Exception, 'value info', self._run_command,
caffe2_to_onnx, args)
args.extend(
['--value-info',
json.dumps({
'X': (TensorProto.FLOAT, (2, 2)),
})])
result = self._run_command(caffe2_to_onnx, args)
onnx_model = ModelProto()
onnx_model.ParseFromString(output.read())
self.assertEqual(len(onnx_model.graph.node), 1)
self.assertEqual(onnx_model.graph.node[0].op_type, 'Relu')
self.assertEqual(len(onnx_model.graph.initializer), 0)
def test_caffe2_to_onnx(self):
caffe2_net = tempfile.NamedTemporaryFile()
caffe2_init_net = tempfile.NamedTemporaryFile()
output = tempfile.NamedTemporaryFile()
model = ModelHelper(name='caffe2-to-onnx-test')
brew.relu(model, ["X"], "Y")
caffe2_net.write(model.net.Proto().SerializeToString())
caffe2_net.flush()
init_model = ModelHelper(name='caffe2-to-onnx-init-test')
init_model.net.GivenTensorFill([], 'X', shape=[2, 2],
values=np.zeros((2, 2)).flatten().astype(float))
caffe2_init_net.write(init_model.net.Proto().SerializeToString())
caffe2_init_net.flush()
result = self._run_command(
caffe2_to_onnx, [
caffe2_net.name,
'--caffe2-init-net', caffe2_init_net.name,
'--output', output.name,
])
onnx_model = onnx_pb2.ModelProto()
onnx_model.ParseFromString(output.read())
self.assertEqual(len(onnx_model.graph.node), 1)
self.assertEqual(onnx_model.graph.node[0].op_type, 'Relu')
self.assertEqual(len(onnx_model.graph.initializer), 1)
self.assertEqual(onnx_model.graph.initializer[0].name, onnx_model.graph.input[0].name)
def test_fast_memonger_unique_outputs(self):
m = model_helper.ModelHelper()
fc = []
for i in range(2):
z = brew.fc(m,
"data{}".format(i),
"fc".format(i),
dim_in=2,
dim_out=2)
fc.append(z)
r = []
# Trick is here to have same input appear twice in a same Sum
for x in fc:
for y in fc:
r.append(brew.sum(m, [x, y], 1))
concated = brew.concat(m, r, "concated")
brew.relu(m, concated, "merged")
static_blobs = \
[o for op in m.param_init_net.Proto().op for o in op.output] + \
["merged"] + ["data{}".format(i) for i in range(len(fc))]
optimized_net = memonger.optimize_inference_fast(
m.Proto(), static_blobs)
for op in optimized_net.op:
self.assertEqual(len(op.output), len(set(op.output)), str(op))
def conv_dw(model, inputs, inp, oup, stride, name):
W_dw = model.create_param(param_name=name + '_dw_w',
shape=[inp, 1, 3, 3],
initializer=initializers.update_initializer(
None, None, ("XavierFill", {})),
tags=ParameterTags.WEIGHT)
h = inputs.Conv([W_dw], [name + '_dw'],
kernel_h=3,
kernel_w=3,
stride_h=stride,
stride_w=stride,
pad_b=1,
pad_l=1,
pad_r=1,
pad_t=1,
order='NCHW',
group=inp)
h = brew.spatial_bn(model, h, name + '_dw_bn', inp, is_test=True)
h = brew.relu(model, h, h)
h = brew.conv(model,
h,
name + '_sep',
dim_in=inp,
dim_out=oup,
kernel=1,
pad=0,
stride=1,
no_bias=True)
h = brew.spatial_bn(model, h, name + '_sep_bn', oup, is_test=True)
h = brew.relu(model, h, h)
return h
def AddNetModel_5(model, data):
# 0 params, dims [1, 70, 182]
input1 = data
# 18 params, dims [2, 68, 180]
conv2 = brew.conv(model,
input1,
'conv2',
dim_in=1,
dim_out=2,
kernel=3,
stride=1)
# 0 params, dims [2, 34, 90]
pool3 = brew.max_pool(model, conv2, 'pool3', kernel=2, stride=2)
# 72 params, dims [4, 32, 88]
conv4 = brew.conv(model,
pool3,
'conv4',
dim_in=2,
dim_out=4,
kernel=3,
stride=1)
# 0 params, dims [4, 16, 44]
pool5 = brew.max_pool(model, conv4, 'pool5', kernel=2, stride=2)
# 288 params, dims [8, 14, 42]
conv6 = brew.conv(model,
pool5,
'conv6',
dim_in=4,
dim_out=8,
kernel=3,
stride=1)
# 0 params, dims [8, 14, 42]
relu7 = brew.relu(model, conv6, 'relu7')
# 64 params, dims [2, 7, 21]
conv8 = brew.conv(model,
relu7,
'conv8',
dim_in=8,
dim_out=2,
kernel=2,
stride=2)
# 0 params, dims [2, 7, 21]
relu9 = brew.relu(model, conv8, 'relu9')
# 784 params, dims [8, 1, 3]
conv10 = brew.conv(model,
relu9,
'conv10',
dim_in=2,
dim_out=8,
kernel=7,
stride=7)
# 0 params, dims [8, 1, 3]
relu11 = brew.relu(model, conv10, 'relu11')
# 192 params, dims [8]
fc12 = brew.fc(model, relu11, 'fc12', dim_in=24, dim_out=8)
# 0 params, dims [8]
relu13 = brew.relu(model, fc12, 'relu13')
# 16 params, dims [2]
output = brew.fc(model, relu13, 'output', dim_in=8, dim_out=2)
return output
def simple_mlp():
model = ModelHelper(name="r")
brew.relu(
model,
brew.fc(model,
brew.relu(model, brew.fc(model, "data", "fc1", 10, 10), "rl1"),
"fc2", 10, 10), "rl2")
return model, [(1, 10)]
def AddLeNetModel(model, data):
fc1 = brew.fc(model, data, 'fc1', dim_in=784, dim_out=1000)
fc1 = brew.relu(model, fc1, fc1)
fc2 = brew.fc(model, fc1, 'fc2', dim_in=1000, dim_out=1000)
fc2 = brew.relu(model, fc2, fc2)
pred = brew.fc(model, fc2, 'fc3', dim_in=1000, dim_out=10)
softmax = brew.softmax(model, pred, 'softmax')
return softmax
def Add_Original_CIFAR10_Model(model,
data,
num_classes,
image_height,
image_width,
image_channels,
save_png=True):
conv1 = brew.conv(model,
data,
'conv1',
dim_in=image_channels,
dim_out=32,
kernel=5,
stride=1,
pad=2)
h, w = update_dims(height=image_height,
width=image_width,
kernel=5,
stride=1,
pad=2)
pool1 = brew.max_pool(model, conv1, 'pool1', kernel=3, stride=2)
h, w = update_dims(height=h, width=w, kernel=3, stride=2, pad=0)
relu1 = brew.relu(model, pool1, 'relu1')
conv2 = brew.conv(model,
relu1,
'conv2',
dim_in=32,
dim_out=32,
kernel=5,
stride=1,
pad=2)
h, w = update_dims(height=h, width=w, kernel=5, stride=1, pad=2)
relu2 = brew.relu(model, conv2, 'relu2')
pool2 = brew.average_pool(model, relu2, 'pool2', kernel=3, stride=2)
h, w = update_dims(height=h, width=w, kernel=3, stride=2, pad=0)
conv3 = brew.conv(model,
pool2,
'conv3',
dim_in=32,
dim_out=64,
kernel=5,
stride=1,
pad=2)
h, w = update_dims(height=h, width=w, kernel=5, stride=1, pad=2)
relu3 = brew.relu(model, conv3, 'relu3')
pool3 = brew.average_pool(model, relu3, 'pool3', kernel=3, stride=2)
h, w = update_dims(height=h, width=w, kernel=3, stride=2, pad=0)
fc1 = brew.fc(model, pool3, 'fc1', dim_in=64 * h * w, dim_out=64)
fc2 = brew.fc(model, fc1, 'fc2', dim_in=64, dim_out=num_classes)
softmax = brew.softmax(model, fc2, 'softmax')
if save_png:
graph = net_drawer.GetPydotGraph(model.net, rankdir="LR")
graph.write_png("CIFAR10_Model.png")
return softmax
def test_mobile_exporter(self):
model = ModelHelper(name="mobile_exporter_test_model")
# Test LeNet
brew.conv(model, 'data', 'conv1', dim_in=1, dim_out=20, kernel=5)
brew.max_pool(model, 'conv1', 'pool1', kernel=2, stride=2)
brew.conv(model, 'pool1', 'conv2', dim_in=20, dim_out=50, kernel=5)
brew.max_pool(model, 'conv2', 'pool2', kernel=2, stride=2)
brew.fc(model, 'pool2', 'fc3', dim_in=50 * 4 * 4, dim_out=500)
brew.relu(model, 'fc3', 'fc3')
brew.fc(model, 'fc3', 'pred', 500, 10)
brew.softmax(model, 'pred', 'out')
# Create our mobile exportable networks
workspace.RunNetOnce(model.param_init_net)
init_net, predict_net = mobile_exporter.Export(
workspace, model.net, model.params
)
# Populate the workspace with data
np_data = np.random.rand(1, 1, 28, 28).astype(np.float32)
workspace.FeedBlob("data", np_data)
workspace.CreateNet(model.net)
workspace.RunNet(model.net)
ref_out = workspace.FetchBlob("out")
# Clear the workspace
workspace.ResetWorkspace()
# Populate the workspace with data
workspace.RunNetOnce(init_net)
# Fake "data" is populated by init_net, we have to replace it
workspace.FeedBlob("data", np_data)
# Overwrite the old net
workspace.CreateNet(predict_net, True)
workspace.RunNet(predict_net.name)
manual_run_out = workspace.FetchBlob("out")
np.testing.assert_allclose(
ref_out, manual_run_out, atol=1e-10, rtol=1e-10
)
# Clear the workspace
workspace.ResetWorkspace()
# Predictor interface test (simulates writing to disk)
predictor = workspace.Predictor(
init_net.SerializeToString(), predict_net.SerializeToString()
)
# Output is a vector of outputs but we only care about the first and only result
predictor_out = predictor.run([np_data])
assert len(predictor_out) == 1
predictor_out = predictor_out[0]
np.testing.assert_allclose(
ref_out, predictor_out, atol=1e-10, rtol=1e-10
)
def testShapeInferenceConvNet(self):
model = model_helper.ModelHelper(name="convtest")
model.NHWC2NCHW("data", "data_nchw")
brew.conv(model,
"data_nchw",
'conv1',
3,
64,
weight_init=("MSRAFill", {}),
kernel=7,
stride=2,
pad=3,
no_bias=0)
brew.spatial_bn(model,
'conv1',
'conv1_spatbn_relu',
64,
epsilon=1e-3,
is_test=False)
brew.relu(model, 'conv1_spatbn_relu', 'conv1_spatbn_relu')
brew.max_pool(model, 'conv1_spatbn_relu', 'pool1', kernel=3, stride=2)
brew.fc(model, 'pool1', 'fc', dim_in=(64 * 56 * 56), dim_out=100)
brew.dropout(model, 'fc', 'fc_drop', is_test=False)
model.Sigmoid('fc_drop', 'fc_sigm')
brew.softmax(model, 'fc_sigm', 'softmax')
model.LabelCrossEntropy(['softmax', 'label'], 'xent')
loss = model.AveragedLoss('xent', 'loss')
model.AddGradientOperators([loss])
LR = model.param_init_net.ConstantFill([], 'LR', shape=[1], value=0.1)
for param in model.GetParams():
param_grad = model.param_to_grad[param]
param_momentum = model.param_init_net.ConstantFill([param],
param +
'_momentum',
value=0.0)
model.net.MomentumSGDUpdate(
[param_grad, param_momentum, LR, param],
[param_grad, param_momentum, param],
)
workspace.FeedBlob(
"data",
np.random.rand(16, 227, 227, 3).astype(np.float32),
)
workspace.FeedBlob(
"label",
(100 * np.random.rand(16)).astype(np.int32),
)
workspace.FeedBlob(
"label",
(100 * np.random.rand(16)).astype(np.int32),
)
# Then do automatic comparison test: run the next once to
# initialize everything
self.InferTensorRunAndCompare(model)
def test_mobile_exporter(self):
model = ModelHelper(name="mobile_exporter_test_model")
# Test LeNet
brew.conv(model, 'data', 'conv1', dim_in=1, dim_out=20, kernel=5)
brew.max_pool(model, 'conv1', 'pool1', kernel=2, stride=2)
brew.conv(model, 'pool1', 'conv2', dim_in=20, dim_out=50, kernel=5)
brew.max_pool(model, 'conv2', 'pool2', kernel=2, stride=2)
brew.fc(model, 'pool2', 'fc3', dim_in=50 * 4 * 4, dim_out=500)
brew.relu(model, 'fc3', 'fc3')
brew.fc(model, 'fc3', 'pred', 500, 10)
brew.softmax(model, 'pred', 'out')
# Create our mobile exportable networks
workspace.RunNetOnce(model.param_init_net)
init_net, predict_net = mobile_exporter.Export(workspace, model.net,
model.params)
# Populate the workspace with data
np_data = np.random.rand(1, 1, 28, 28).astype(np.float32)
workspace.FeedBlob("data", np_data)
workspace.CreateNet(model.net)
workspace.RunNet(model.net)
ref_out = workspace.FetchBlob("out")
# Clear the workspace
workspace.ResetWorkspace()
# Populate the workspace with data
workspace.RunNetOnce(init_net)
# Fake "data" is populated by init_net, we have to replace it
workspace.FeedBlob("data", np_data)
# Overwrite the old net
workspace.CreateNet(predict_net, True)
workspace.RunNet(predict_net.name)
manual_run_out = workspace.FetchBlob("out")
np.testing.assert_allclose(ref_out,
manual_run_out,
atol=1e-10,
rtol=1e-10)
# Clear the workspace
workspace.ResetWorkspace()
# Predictor interface test (simulates writing to disk)
predictor = workspace.Predictor(init_net.SerializeToString(),
predict_net.SerializeToString())
# Output is a vector of outputs but we only care about the first and only result
predictor_out = predictor.run([np_data])
assert len(predictor_out) == 1
predictor_out = predictor_out[0]
np.testing.assert_allclose(ref_out,
predictor_out,
atol=1e-10,
rtol=1e-10)
def AddLeNetModel(model, data, device_opts):
'''
This part is the standard LeNet model: from data to the softmax prediction.
For each convolutional layer we specify dim_in - number of input channels
and dim_out - number or output channels. Also each Conv and MaxPool layer changes the
image size. For example, kernel of size 5 reduces each side of an image by 4.
While when we have kernel and stride sizes equal 2 in a MaxPool layer, it divides
each side in half.
'''
with core.DeviceScope(device_opts):
############# Change dim_in value if images are more than 1 color channel
# Image size: 64x64
conv1 = brew.conv(model, data, 'conv1', dim_in=1, dim_out=32, kernel=5)
h, w = update_dims(height=image_height,
width=image_width,
kernel=5,
stride=1,
pad=0)
# Image size: 60x60
pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2)
h, w = update_dims(height=h, width=w, kernel=2, stride=2, pad=0)
relu1 = brew.relu(model, pool1, 'relu1')
# Image size: 30x30
conv2 = brew.conv(model,
relu1,
'conv2',
dim_in=32,
dim_out=64,
kernel=5)
h, w = update_dims(height=h, width=w, kernel=5, stride=1, pad=0)
# Image size: 26x26
pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2)
h, w = update_dims(height=h, width=w, kernel=2, stride=2, pad=0)
relu2 = brew.relu(model, pool2, 'relu2')
# Image size: 13x13
conv3 = brew.conv(model,
relu2,
'conv3',
dim_in=64,
dim_out=64,
kernel=5)
h, w = update_dims(height=h, width=w, kernel=5, stride=1, pad=0)
# Image size: 9x9
pool3 = brew.max_pool(model, conv3, 'pool3', kernel=2, stride=2)
h, w = update_dims(height=h, width=w, kernel=2, stride=2, pad=0)
relu3 = brew.relu(model, pool3, 'relu3')
# Image size: 4x4
# 50 * 4 * 4 stands for dim_out from previous layer multiplied by the image size
############# Change dim_in value if images are not 28x28
fc3 = brew.fc(model, relu3, 'fc3', dim_in=64 * h * w, dim_out=500)
fc3 = brew.relu(model, fc3, fc3)
pred = brew.fc(model, fc3, 'pred', 500, num_classes)
softmax = brew.softmax(model, pred, 'softmax')
return softmax
def simple_cnn():
model = ModelHelper(name="r", arg_scope={"order": "NCHW", "is_test": True})
brew.conv(
model, "data", 'conv1', 3, 16, kernel=3, stride=1
)
brew.spatial_bn(
model, 'conv1', 'conv1_spatbn', 16, epsilon=1e-3
)
brew.relu(model, 'conv1_spatbn', 'relu1')
return model, (1, 3, 32, 32)
def create_resnet(
model,
data,
num_input_channels,
num_groups,
num_labels,
device_opts,
is_test=False,
):
with core.DeviceScope(device_opts):
filters = [16, 32, 64]
brew.conv(model,
data,
'conv1',
num_input_channels,
filters[0],
no_bias=1,
kernel=3,
stride=1,
pad=1)
builder = ResNetBuilder(model, 'conv1', no_bias=1, is_test=is_test)
# input: 32x32x16 output: 32x32x16
for _ in range(num_groups):
builder.add_simple_block(filters[0],
filters[0],
down_sampling=False)
# input: 32x32x16 output: 16x16x32
builder.add_simple_block(filters[0], filters[1], down_sampling=True)
for _ in range(1, num_groups):
builder.add_simple_block(filters[1],
filters[1],
down_sampling=False)
# input: 16x16x32 output: 8x8x64
builder.add_simple_block(filters[1], filters[2], down_sampling=True)
for _ in range(1, num_groups):
builder.add_simple_block(filters[2],
filters[2],
down_sampling=False)
brew.spatial_bn(model,
builder.prev_blob,
'last_spatbn',
filters[2],
epsilon=1e-3,
is_test=is_test)
brew.relu(model, 'last_spatbn', 'last_relu')
# Final layers
brew.average_pool(model, 'last_relu', 'final_avg', kernel=8, stride=1)
last_out = brew.fc(model, 'final_avg', 'last_out', 64, num_labels)
return last_out
def AddMLP(model, data, batch_size):
'''
Implement MLP model on MNIST
'''
num_units = 4096 # number of nuerons in fc layer
num_labels = 10 # for 10 classes in mnist
drop1 = brew.dropout(model, data, 'drop1', ratio=0.5, is_test=0)
fc2 = brew.fc(model, drop1, 'fc2', dim_in=1 * 28 * 28, dim_out=num_units)
model.Reshape([fc2], [fc2, 'fc2_old_shape'],
shape=(batch_size, num_units, 1, 1))
bn2 = brew.spatial_bn(model,
fc2,
'bn2',
num_units,
epsilon=1e-4,
momentum=0.9)
relu2 = brew.relu(model, bn2, 'relu2')
fc3 = brew.fc(model, relu2, 'fc3', dim_in=num_units, dim_out=num_units)
model.Reshape([fc3], [fc3, 'fc3_old_shape'],
shape=(batch_size, num_units, 1, 1))
bn3 = brew.spatial_bn(model,
fc3,
'bn3',
num_units,
epsilon=1e-4,
momentum=0.9)
relu3 = brew.relu(model, bn3, 'relu3')
fc4 = brew.fc(model, relu3, 'fc4', dim_in=num_units, dim_out=num_units)
model.Reshape([fc4], [fc4, 'fc4_old_shape'],
shape=(batch_size, num_units, 1, 1))
bn4 = brew.spatial_bn(model,
fc4,
'bn4',
num_units,
epsilon=1e-4,
momentum=0.9)
relu4 = brew.relu(model, bn4, 'relu4')
fc5 = brew.fc(model, relu4, 'fc5', dim_in=num_units, dim_out=num_labels)
model.Reshape([fc5], [fc5, 'fc5_old_shape'],
shape=(batch_size, num_labels, 1, 1))
bn5 = brew.spatial_bn(model,
fc5,
'bn5',
num_labels,
epsilon=1e-4,
momentum=0.9)
softmax = brew.softmax(model, bn5, 'softmax')
return softmax
def AddUpgradedLeNetModel_GPU(model, data, num_classes, image_height,
image_width, image_channels, device_opts):
with core.DeviceScope(device_opts):
# Image size: 64x64
conv1 = brew.conv(model,
data,
'conv1',
dim_in=image_channels,
dim_out=32,
kernel=5)
h, w = update_dims(height=image_height,
width=image_width,
kernel=5,
stride=1,
pad=0)
# Image size: 60x60
pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2)
h, w = update_dims(height=h, width=w, kernel=2, stride=2, pad=0)
relu1 = brew.relu(model, pool1, 'relu1')
# Image size: 30x30
conv2 = brew.conv(model,
relu1,
'conv2',
dim_in=32,
dim_out=64,
kernel=5)
h, w = update_dims(height=h, width=w, kernel=5, stride=1, pad=0)
# Image size: 26x26
pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2)
h, w = update_dims(height=h, width=w, kernel=2, stride=2, pad=0)
relu2 = brew.relu(model, pool2, 'relu2')
# Image size: 13x13
conv3 = brew.conv(model,
relu2,
'conv3',
dim_in=64,
dim_out=64,
kernel=5)
h, w = update_dims(height=h, width=w, kernel=5, stride=1, pad=0)
# Image size: 9x9
pool3 = brew.max_pool(model, conv3, 'pool3', kernel=2, stride=2)
h, w = update_dims(height=h, width=w, kernel=2, stride=2, pad=0)
relu3 = brew.relu(model, pool3, 'relu3')
# Image size: 4x4
# 50 * 4 * 4 stands for dim_out from previous layer multiplied by the image size
############# Change dim_in value if images are not 28x28
fc3 = brew.fc(model, relu3, 'fc3', dim_in=64 * h * w, dim_out=500)
fc3 = brew.relu(model, fc3, fc3)
pred = brew.fc(model, fc3, 'pred', 500, num_classes)
softmax = brew.softmax(model, pred, 'softmax')
return softmax
def AddLeNetModel(model, data):
'''
This part is the standard LeNet model: from data to the softmax prediction.
For each convolutional layer we specify dim_in - number of input channels
and dim_out - number or output channels. Also each Conv and MaxPool layer changes the
image size. For example, kernel of size 5 reduces each side of an image by 4.
While when we have kernel and stride sizes equal 2 in a MaxPool layer, it divides
each side in half.
'''
# Image size: 28 x 28 -> 24 x 24
conv1 = brew.conv(model, data, 'conv1', dim_in=1, dim_out=20, kernel=5)
# Image size: 24 x 24 -> 12 x 12
pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2)
# Image size: 12 x 12 -> 8 x 8
conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=50, kernel=5)
# Image size: 8 x 8 -> 4 x 4
pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2)
# 50 * 4 * 4 stands for dim_out from previous layer multiplied by the image size
fc3 = brew.fc(model, pool2, 'fc3', dim_in=50 * 4 * 4, dim_out=500)
fc3 = brew.relu(model, fc3, fc3)
pred = brew.fc(model, fc3, 'pred', 500, 10)
softmax = brew.softmax(model, pred, 'softmax')
return softmax
def AddLeNetModel(model, data, save_png=True):
"""This part is the standard LeNet model: from data to the softmax prediction.
For each convolutional layer we specify dim_in - number of input channels
and dim_out - number or output channels. Also each Conv and MaxPool layer changes the
image size. For example, kernel of size 5 reduces each side of an image by 4.
While when we have kernel and stride sizes equal 2 in a MaxPool layer, it divides
each side in half.
"""
conv1 = brew.conv(model, data, 'conv1', dim_in=1, dim_out=20, kernel=5)
pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2)
conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=50, kernel=5)
pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2)
fc3 = brew.fc(model, pool2, 'fc3', dim_in=50 * 4 * 4, dim_out=500)
relu3 = brew.relu(model, fc3, 'relu3')
pred = brew.fc(model, relu3, 'pred', dim_in=500, dim_out=10)
softmax = brew.softmax(model, pred, 'softmax')
if save_png:
graph = net_drawer.GetPydotGraph(model.net)
graph.write_png("LeNet.png")
return softmax
def meta_conv(
model,
inputs,
dim_in=64,
dim_out=64,
kernel=3,
pad=1,
stride=1,
module_seq=None,
conv_seq=None,
):
# convolution layer
conv = brew.conv(model,
inputs,
'conv{}_{}'.format(module_seq, conv_seq),
dim_in=dim_in,
dim_out=dim_out,
kernel=kernel,
stride=stride,
pad=pad)
# ReLU layer
relu = brew.relu(
model,
conv,
conv # in-place
)
return relu
def add_lenet_conv_topk(model, data, **kwargs):
'''
Based on original LeNet structure, according to meProp, we add topk
gradient selecting layer for conv layers
'''
# resolving arguments
config = kwargs.get('config', {})
k = kwargs.get('k', 10)
# Image size: 28 x 28 -> 24 x 24
conv1 = brew.conv(model, data, 'conv1', dim_in=3, dim_out=20, kernel=5)
conv1_hook = model.net.TopKGradHook(conv1, 'conv1_hook', k=k)
# Image size: 24 x 24 -> 12 x 12
pool1 = brew.max_pool(model, conv1_hook, 'pool1', kernel=2, stride=2)
# Image size: 12 x 12 -> 8 x 8
conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=50, kernel=5)
conv2_hook = model.net.TopKGradHook(conv2, 'conv2_hook', k=k)
# Image size: 8 x 8 -> 4 x 4
pool2 = brew.max_pool(model, conv2_hook, 'pool2', kernel=2, stride=2)
# 50 * 4 * 4 stands for dim_out from previous layer multiplied by the image size
fc3 = brew.fc(model, pool2, 'fc3', dim_in=50 * 4 * 4, dim_out=500)
relu = brew.relu(model, fc3, fc3)
pred = brew.fc(model, relu, 'pred', 500, config['model_arch']['num_classes'])
softmax = brew.softmax(model, pred, 'softmax')
return softmax
def AddLeNetModel(model, data):
'''
This part is the standard LeNet model: from data to the softmax prediction.
For each convolutional layer we specify dim_in - number of input channels
and dim_out - number or output channels. Also each Conv and MaxPool layer changes the
image size. For example, kernel of size 5 reduces each side of an image by 4.
While when we have kernel and stride sizes equal 2 in a MaxPool layer, it divides
each side in half.
'''
# Image size: 28 x 28 -> 24 x 24
conv1 = brew.conv(model, data, 'conv1', dim_in=1, dim_out=20, kernel=5)
# Image size: 24 x 24 -> 12 x 12
pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2)
# Image size: 12 x 12 -> 8 x 8
conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=50, kernel=5)
# Image size: 8 x 8 -> 4 x 4
pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2)
# 50 * 4 * 4 stands for dim_out from previous layer multiplied by the image size
fc3 = brew.fc(model, pool2, 'fc3', dim_in=50 * 4 * 4, dim_out=500)
fc3 = brew.relu(model, fc3, fc3)
pred = brew.fc(model, fc3, 'pred', 500, 10)
softmax = brew.softmax(model, pred, 'softmax')
return softmax
def add_lenet_fc_topk(model, data, **kwargs):
'''
Based on original LeNet structure, according to meProp, we add topk
gradient selecting layer for fc layers
'''
# resolving arguments
k = kwargs.get('k', 10)
# Image size: 28 x 28 -> 24 x 24
conv1 = brew.conv(model, data, 'conv1', dim_in=3, dim_out=20, kernel=5)
# Image size: 24 x 24 -> 12 x 12
pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2)
# Image size: 12 x 12 -> 8 x 8
conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=50, kernel=5)
# Image size: 8 x 8 -> 4 x 4
pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2)
# DEBUG PRINT
model.net.Print(pool2, [])
# set topkgradhook for the hidden fc layers
fc3 = brew.fc(model, pool2, 'fc3', dim_in=50 * 4 * 4, dim_out=500)
fc3_hook = model.net.TopKGradHook(fc3, 'fc3_hook', k=k)
relu = brew.relu(model, fc3_hook, fc3_hook)
# don't set topkgradhook for the output fc layer
pred = brew.fc(model, relu, 'pred', 500, 10)
softmax = brew.softmax(model, pred, 'softmax')
return softmax
def add_relu(self):
self.prev_blob = brew.relu(
self.model,
self.prev_blob,
self.prev_blob, # in-place
)
return self.prev_blob
def AddNetModel_3(model, data):
# 182x70x1 -> 90x34x3
conv1 = brew.conv(model, data, 'conv1', dim_in=1, dim_out=6, kernel=3)
pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2)
# 90x34x3 -> 44x16x5
conv2 = brew.conv(model, pool1, 'conv2', dim_in=6, dim_out=8, kernel=3)
pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2)
# 44x16x5 -> 21x7x7
conv3 = brew.conv(model, pool2, 'conv3', dim_in=8, dim_out=10, kernel=3)
pool3 = brew.max_pool(model, conv3, 'pool3', kernel=2, stride=2)
relu4 = brew.relu(model, pool3, 'relu4')
# 21x7x7 -> 2
fc5 = brew.fc(model, relu4, 'fc5', dim_in=21 * 7 * 10, dim_out=32)
relu5 = brew.relu(model, fc5, 'relu5')
output = brew.fc(model, relu5, 'output', dim_in=32, dim_out=2)
return output
def AddLeNetModel(model, data):
'''
This part is the standard LeNet model: from data to the softmax prediction.
For each convolutional layer we specify dim_in - number of input channels
and dim_out - number or output channels. Also each Conv and MaxPool layer changes the
image size. For example, kernel of size 5 reduces each side of an image by 4.
While when we have kernel and stride sizes equal 2 in a MaxPool layer, it divides
each side in half.
Para encadenar layers, la dimension de salida tiene que coincidir con a dimension de entrada
de la siguiente layer (sin contar las pool, solo las conv)
'''
#Each layer is created using brew, which in turn is using its operator hooks to instantiate each Op.
# Image size: 28 x 28 -> 24 x 24
conv1 = brew.conv(model, data, 'conv1', dim_in=1, dim_out=20, kernel=5)
# Image size: 24 x 24 -> 12 x 12
pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2)
# Image size: 12 x 12 -> 8 x 8
conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=100, kernel=5)
# Image size: 8 x 8 -> 4 x 4
pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2)
# 100 * 4 * 4 stands for dim_out from previous layer multiplied by the image size
fc3 = brew.fc(model, pool2, 'fc3', dim_in=100 * 4 * 4, dim_out=500)
#TODO: RELU despues de cada conv layer -> no mejora mucho la accuracy
relu = brew.relu(model, fc3, fc3)
pred = brew.fc(model, relu, 'pred', 500, 10)
softmax = brew.softmax(model, pred, 'softmax')
return softmax
def conv_factory(self, model, v, num_in_channels, num_filters, kernel,
stride=1, pad=0, relu=True, name='conv'):
"""Standard convolution block: Conv -> BatchNorm -> Activation
"""
if isinstance(pad, int):
pad_t = pad_b = pad_l = pad_r = pad
elif isinstance(pad, list) or isinstance(pad, tuple):
if len(pad) == 2:
pad_t = pad_b = pad[0]
pad_l = pad_r = pad[1]
elif len(pad) == 4:
pad_t = pad[0]
pad_b = pad[1]
pad_l = pad[2]
pad_r = pad[3]
else:
assert False, "Invalid length of pad array. Expecting 2 or 4 but have: " + str(pad)
else:
assert False, "Invalid type of padding: " + str(pad)
v = brew.conv(model, v, name + '_conv', num_in_channels, num_filters,
kernel=kernel, pad_t=pad_t, pad_l=pad_l, pad_b=pad_b,
pad_r=pad_r, stride=stride)
v = brew.spatial_bn(model, v, name+'_bn', num_filters, eps=2e-5,
momentum=0.9, is_test=(self.phase == 'inference'))
if relu is True:
v = brew.relu(model, v, name + '_relu')
return v
def AddLeNetModel(model, data):
'''
This part is the standard LeNet model: from data to the softmax prediction.
For each convolutional layer we specify dim_in - number of input channels
and dim_out - number or output channels. Also each Conv and MaxPool layer changes the
image size. For example, kernel of size 5 reduces each side of an image by 4.
While when we have kernel and stride sizes equal 2 in a MaxPool layer, it divides
each side in half.
'''
# Image size: 28 x 28 -> 24 x 24 / 350 x 350 -> 346 x 346
conv1 = brew.conv(model, data, 'conv1', dim_in=1, dim_out=20, kernel=3)
# Image size: 346 x 346 -> 173 x 173
pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2)
# Image size: 174 x 174
conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=50, kernel=5)
# Image size: 174 x 174 -> 87 x 87
pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2)
# 50 * 4 * 4 stands for dim_out from previous layer multiplied by the image size
# Here, the data is flattened from a tensor of dimension 50x4x4 to a vector of length 50*4*4
fc3 = brew.fc(model, pool2, 'fc3', dim_in=50 * 87 * 87, dim_out=500)
relu3 = brew.relu(model, fc3, 'relu3')
# Last FC Layer
pred = brew.fc(model, relu3, 'pred', dim_in=500, dim_out=10)
# Softmax Layer
softmax = brew.softmax(model, pred, 'softmax')
return softmax
def AddLeNetModel(model, data, num_classes, image_height, image_width,
image_channels):
# Image size: 28 x 28 -> 24 x 24
############# Change dim_in value if images are more than 1 color channel
conv1 = brew.conv(model, data, 'conv1', dim_in=1, dim_out=20, kernel=5)
h, w = update_dims(height=image_height,
width=image_width,
kernel=5,
stride=1,
pad=0)
# Image size: 24 x 24 -> 12 x 12
pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2)
h, w = update_dims(height=h, width=w, kernel=2, stride=2, pad=0)
# Image size: 12 x 12 -> 8 x 8
conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=50, kernel=5)
h, w = update_dims(height=h, width=w, kernel=5, stride=1, pad=0)
# Image size: 8 x 8 -> 4 x 4
pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2)
h, w = update_dims(height=h, width=w, kernel=2, stride=2, pad=0)
# 50 * 4 * 4 stands for dim_out from previous layer multiplied by the image size
############# Change dim_in value if images are not 28x28
fc3 = brew.fc(model, pool2, 'fc3', dim_in=50 * h * w, dim_out=500)
fc3 = brew.relu(model, fc3, fc3)
pred = brew.fc(model, fc3, 'pred', 500, num_classes)
softmax = brew.softmax(model, pred, 'softmax')
return softmax
def AddNet(model, data):
# image size: 28x28 -> 24x24
conv1 = brew.conv(model,
data,
"conv1",
dim_in=1,
dim_out=20,
kernel=5,
stride=1)
# image size: 24x24 -> 12x12
pool1 = brew.max_pool(model, conv1, "pool1", kernel=2, stride=2)
# image size: 12x12 -> 8x8
conv2 = brew.conv(model,
pool1,
"conv2",
dim_in=20,
dim_out=100,
kernel=5,
stride=1)
# image size: 8x8 -> 4x4
pool2 = brew.max_pool(model, conv2, "pool2", kernel=2, stride=2)
# image size: 100x4x4 -> 500
fc3 = brew.fc(model, pool2, "fc3", dim_in=100 * 4 * 4, dim_out=500)
relu = brew.relu(model, fc3, fc3)
# image size: 500 -> 10
fc4 = brew.fc(model, fc3, "fc4", dim_in=500, dim_out=10)
# softmax
softmax = brew.softmax(model, fc4, "softmax")
return softmax
def test_caffe2_simple_model(self):
model = ModelHelper(name="mnist")
# how come those inputs don't break the forward pass =.=a
workspace.FeedBlob("data", np.random.randn(1, 3, 64, 64).astype(np.float32))
workspace.FeedBlob("label", np.random.randn(1, 1000).astype(np.int))
with core.NameScope("conv1"):
conv1 = brew.conv(model, "data", 'conv1', dim_in=1, dim_out=20, kernel=5)
# Image size: 24 x 24 -> 12 x 12
pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2)
# Image size: 12 x 12 -> 8 x 8
conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=100, kernel=5)
# Image size: 8 x 8 -> 4 x 4
pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2)
with core.NameScope("classifier"):
# 50 * 4 * 4 stands for dim_out from previous layer multiplied by the image size
fc3 = brew.fc(model, pool2, 'fc3', dim_in=100 * 4 * 4, dim_out=500)
relu = brew.relu(model, fc3, fc3)
pred = brew.fc(model, relu, 'pred', 500, 10)
softmax = brew.softmax(model, pred, 'softmax')
xent = model.LabelCrossEntropy([softmax, "label"], 'xent')
# compute the expected loss
loss = model.AveragedLoss(xent, "loss")
model.net.RunAllOnMKL()
model.param_init_net.RunAllOnMKL()
model.AddGradientOperators([loss], skip=1)
blob_name_tracker = {}
graph = c2_graph.model_to_graph_def(
model,
blob_name_tracker=blob_name_tracker,
shapes={},
show_simplified=False,
)
compare_proto(graph, self)
def add_relu(self):
self.prev_blob = brew.relu(
self.model,
self.prev_blob,
self.prev_blob, # in-place
)
return self.prev_blob
def testShapeInferenceConvNet(self):
model = model_helper.ModelHelper(name="convtest")
model.NHWC2NCHW("data", "data_nchw")
brew.conv(model, "data_nchw", 'conv1', 3, 64,
weight_init=("MSRAFill", {}), kernel=7,
stride=2, pad=3, no_bias=0)
brew.spatial_bn(model, 'conv1', 'conv1_spatbn_relu', 64, epsilon=1e-3, is_test=False)
brew.relu(model, 'conv1_spatbn_relu', 'conv1_spatbn_relu')
brew.max_pool(model, 'conv1_spatbn_relu', 'pool1', kernel=3, stride=2)
brew.fc(model, 'pool1', 'fc', dim_in=(64 * 56 * 56), dim_out=100)
brew.dropout(model, 'fc', 'fc_drop', is_test=False)
model.Sigmoid('fc_drop', 'fc_sigm')
brew.softmax(model, 'fc_sigm', 'softmax')
model.LabelCrossEntropy(['softmax', 'label'], 'xent')
loss = model.AveragedLoss('xent', 'loss')
model.AddGradientOperators([loss])
LR = model.param_init_net.ConstantFill(
[], 'LR', shape=[1], value=0.1
)
for param in model.GetParams():
param_grad = model.param_to_grad[param]
param_momentum = model.param_init_net.ConstantFill(
[param], param + '_momentum', value=0.0
)
model.net.MomentumSGDUpdate(
[param_grad, param_momentum, LR, param],
[param_grad, param_momentum, param],
)
workspace.FeedBlob(
"data",
np.random.rand(16, 227, 227, 3).astype(np.float32),
)
workspace.FeedBlob(
"label",
(100 * np.random.rand(16)).astype(np.int32),
)
workspace.FeedBlob(
"label",
(100 * np.random.rand(16)).astype(np.int32),
)
# Then do automatic comparison test: run the next once to
# initialize everything
self.InferTensorRunAndCompare(model)
def test_relu(self):
Xpos = np.ones((5, 5)).astype(np.float32) - 0.5
Xneg = np.ones((5, 5)).astype(np.float32) - 1.5
workspace.FeedBlob("xpos", Xpos)
workspace.FeedBlob("xneg", Xneg)
model = ModelHelper(name="test_model")
brew.relu(model, "xpos", "out_xpos")
brew.relu(model, "xneg", "out_xneg")
model.Validate()
workspace.RunNetOnce(model.param_init_net)
workspace.RunNetOnce(model.net)
pos = workspace.FetchBlob("out_xpos")
self.assertAlmostEqual(pos.mean(), 0.5)
neg = workspace.FetchBlob("out_xneg")
self.assertAlmostEqual(neg.mean(), 0)
def simple_mlp():
model = ModelHelper(name="r")
brew.relu(
model,
brew.fc(
model,
brew.relu(
model,
brew.fc(
model,
"data",
"fc1",
10,
10),
"rl1"),
"fc2",
10,
10),
"rl2")
return model, (1, 10)
def test_fast_memonger_unique_outputs(self):
m = model_helper.ModelHelper()
fc = []
for i in range(2):
z = brew.fc(
m, "data{}".format(i), "fc".format(i), dim_in=2, dim_out=2)
fc.append(z)
r = []
# Trick is here to have same input appear twice in a same Sum
for x in fc:
for y in fc:
r.append(brew.sum(m, [x, y], 1))
concated = brew.concat(m, r, "concated")
brew.relu(m, concated, "merged")
static_blobs = \
[o for op in m.param_init_net.Proto().op for o in op.output] + \
["merged"] + ["data{}".format(i) for i in range(len(fc))]
optimized_net = memonger.optimize_inference_fast(
m.Proto(), static_blobs)
for op in optimized_net.op:
self.assertEqual(len(op.output), len(set(op.output)), str(op))
def create_resnet_32x32(
model, data, num_input_channels, num_groups, num_labels, is_test=False
):
'''
Create residual net for smaller images (sec 4.2 of He et. al (2015))
num_groups = 'n' in the paper
'''
# conv1 + maxpool
brew.conv(
model, data, 'conv1', num_input_channels, 16, kernel=3, stride=1
)
brew.spatial_bn(
model, 'conv1', 'conv1_spatbn', 16, epsilon=1e-3, is_test=is_test
)
brew.relu(model, 'conv1_spatbn', 'relu1')
# Number of blocks as described in sec 4.2
filters = [16, 32, 64]
builder = ResNetBuilder(model, 'relu1', is_test=is_test)
prev_filters = 16
for groupidx in range(0, 3):
for blockidx in range(0, 2 * num_groups):
builder.add_simple_block(
prev_filters if blockidx == 0 else filters[groupidx],
filters[groupidx],
down_sampling=(True if blockidx == 0 and
groupidx > 0 else False))
prev_filters = filters[groupidx]
# Final layers
brew.average_pool(
model, builder.prev_blob, 'final_avg', kernel=8, stride=1
)
brew.fc(model, 'final_avg', 'last_out', 64, num_labels)
softmax = brew.softmax(model, 'last_out', 'softmax')
return softmax
def test_tensorboard_graphs(self):
model = model_helper.ModelHelper(name="overfeat")
data, label = brew.image_input(
model, ["db"], ["data", "label"], is_test=0
)
with core.NameScope("conv1"):
conv1 = brew.conv(model, data, "conv1", 3, 96, 11, stride=4)
relu1 = brew.relu(model, conv1, conv1)
pool1 = brew.max_pool(model, relu1, "pool1", kernel=2, stride=2)
with core.NameScope("classifier"):
fc = brew.fc(model, pool1, "fc", 4096, 1000)
pred = brew.softmax(model, fc, "pred")
xent = model.LabelCrossEntropy([pred, label], "xent")
loss = model.AveragedLoss(xent, "loss")
model.AddGradientOperators([loss], skip=1)
c2_dir = tempfile.mkdtemp()
tf_dir = tempfile.mkdtemp()
with open(os.path.join(c2_dir, "init"), "w") as f:
f.write(str(model.param_init_net.Proto()))
with open(os.path.join(c2_dir, "net"), "w") as f:
f.write(str(model.net.Proto()))
runner = click.testing.CliRunner()
result = runner.invoke(
tb.cli,
["tensorboard-graphs",
"--c2-netdef", os.path.join(c2_dir, "init"),
"--c2-netdef", os.path.join(c2_dir, "net"),
"--tf-dir", tf_dir])
self.assertEqual(result.exit_code, 0)
entries = list(os.walk(tf_dir))
self.assertEqual(len(entries), 1)
((d, _, (fname,)),) = entries
self.assertEqual(tf_dir, d)
events = list(tf.train.summary_iterator(os.path.join(tf_dir, fname)))
self.assertEqual(len(events), 3)
events = events[1:]
nets = [model.param_init_net, model.net]
for i, (event, net) in enumerate(zip(events, nets), start=1):
self.assertEqual(event.step, i)
self.assertEqual(event.wall_time, i)
g = tf.GraphDef()
g.ParseFromString(event.graph_def)
self.assertMultiLineEqual(
str(g),
str(tb_exporter.nets_to_graph_def([net])))
def AlexNet(order, cudnn_ws):
my_arg_scope = {
'order': order,
'use_cudnn': True,
'cudnn_exhaustive_search': True,
}
if cudnn_ws:
my_arg_scope['ws_nbytes_limit'] = cudnn_ws
model = model_helper.ModelHelper(
name="alexnet",
arg_scope=my_arg_scope,
)
conv1 = brew.conv(
model,
"data",
"conv1",
3,
64,
11, ('XavierFill', {}), ('ConstantFill', {}),
stride=4,
pad=2
)
relu1 = brew.relu(model, conv1, "conv1")
pool1 = brew.max_pool(model, relu1, "pool1", kernel=3, stride=2)
conv2 = brew.conv(
model,
pool1,
"conv2",
64,
192,
5,
('XavierFill', {}),
('ConstantFill', {}),
pad=2
)
relu2 = brew.relu(model, conv2, "conv2")
pool2 = brew.max_pool(model, relu2, "pool2", kernel=3, stride=2)
conv3 = brew.conv(
model,
pool2,
"conv3",
192,
384,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1
)
relu3 = brew.relu(model, conv3, "conv3")
conv4 = brew.conv(
model,
relu3,
"conv4",
384,
256,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1
)
relu4 = brew.relu(model, conv4, "conv4")
conv5 = brew.conv(
model,
relu4,
"conv5",
256,
256,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1
)
relu5 = brew.relu(model, conv5, "conv5")
pool5 = brew.max_pool(model, relu5, "pool5", kernel=3, stride=2)
fc6 = brew.fc(
model,
pool5, "fc6", 256 * 6 * 6, 4096, ('XavierFill', {}),
('ConstantFill', {})
)
relu6 = brew.relu(model, fc6, "fc6")
fc7 = brew.fc(
model, relu6, "fc7", 4096, 4096, ('XavierFill', {}), ('ConstantFill', {})
)
relu7 = brew.relu(model, fc7, "fc7")
fc8 = brew.fc(
model, relu7, "fc8", 4096, 1000, ('XavierFill', {}), ('ConstantFill', {})
)
pred = brew.softmax(model, fc8, "pred")
xent = model.net.LabelCrossEntropy([pred, "label"], "xent")
model.net.AveragedLoss(xent, "loss")
return model, 224
def simple_relu():
model = ModelHelper(name="r")
brew.relu(model, "data", "fc")
return model, (1, 10)
def _InceptionModule(
model, input_blob, input_depth, output_name, conv1_depth, conv3_depths,
conv5_depths, pool_depth
):
# path 1: 1x1 conv
conv1 = brew.conv(
model, input_blob, output_name + ":conv1", input_depth, conv1_depth, 1,
('XavierFill', {}), ('ConstantFill', {})
)
conv1 = brew.relu(model, conv1, conv1)
# path 2: 1x1 conv + 3x3 conv
conv3_reduce = brew.conv(
model, input_blob, output_name + ":conv3_reduce", input_depth,
conv3_depths[0], 1, ('XavierFill', {}), ('ConstantFill', {})
)
conv3_reduce = brew.relu(model, conv3_reduce, conv3_reduce)
conv3 = brew.conv(
model,
conv3_reduce,
output_name + ":conv3",
conv3_depths[0],
conv3_depths[1],
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1
)
conv3 = brew.relu(model, conv3, conv3)
# path 3: 1x1 conv + 5x5 conv
conv5_reduce = brew.conv(
model, input_blob, output_name + ":conv5_reduce", input_depth,
conv5_depths[0], 1, ('XavierFill', {}), ('ConstantFill', {})
)
conv5_reduce = brew.relu(model, conv5_reduce, conv5_reduce)
conv5 = brew.conv(
model,
conv5_reduce,
output_name + ":conv5",
conv5_depths[0],
conv5_depths[1],
5,
('XavierFill', {}),
('ConstantFill', {}),
pad=2
)
conv5 = brew.relu(model, conv5, conv5)
# path 4: pool + 1x1 conv
pool = brew.max_pool(
model,
input_blob,
output_name + ":pool",
kernel=3,
stride=1,
pad=1
)
pool_proj = brew.conv(
model, pool, output_name + ":pool_proj", input_depth, pool_depth, 1,
('XavierFill', {}), ('ConstantFill', {})
)
pool_proj = brew.relu(model, pool_proj, pool_proj)
output = brew.concat(model, [conv1, conv3, conv5, pool_proj], output_name)
return output
def Relu(self, *args, **kwargs):
return brew.relu(
self, *args, order=self.order, use_cudnn=self.use_cudnn, **kwargs
)
def OverFeat(order, cudnn_ws, device):
my_arg_scope = {'order': order, 'use_cudnn': True,
'cudnn_exhaustive_search': True,
'ws_nbytes_limit': str(cudnn_ws)}
model = ModelHelper(name='overfeat', arg_scope=my_arg_scope)
conv1 = brew.conv(
model,
"data",
"conv1",
3,
96,
11,
('XavierFill', {}),
('ConstantFill', {}),
stride=4
)
relu1 = brew.relu(model, conv1, "conv1")
pool1 = brew.max_pool(model, relu1, "pool1", kernel=2, stride=2)
conv2 = brew.conv(
model, pool1, "conv2", 96, 256, 5, ('XavierFill', {}), ('ConstantFill', {})
)
relu2 = brew.relu(model, conv2, "conv2")
pool2 = brew.max_pool(model, relu2, "pool2", kernel=2, stride=2)
conv3 = brew.conv(
model,
pool2,
"conv3",
256,
512,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1
)
relu3 = brew.relu(model, conv3, "conv3")
conv4 = brew.conv(
model,
relu3,
"conv4",
512,
1024,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1
)
relu4 = brew.relu(model, conv4, "conv4")
conv5 = brew.conv(
model,
relu4,
"conv5",
1024,
1024,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1
)
relu5 = brew.relu(model, conv5, "conv5")
pool5 = brew.max_pool(model, relu5, "pool5", kernel=2, stride=2)
fc6 = brew.fc(
model, pool5, "fc6", 1024 * 6 * 6, 3072, ('XavierFill', {}),
('ConstantFill', {})
)
relu6 = brew.relu(model, fc6, "fc6")
fc7 = brew.fc(
model, relu6, "fc7", 3072, 4096, ('XavierFill', {}), ('ConstantFill', {})
)
relu7 = brew.relu(model, fc7, "fc7")
fc8 = brew.fc(
model, relu7, "fc8", 4096, 1000, ('XavierFill', {}), ('ConstantFill', {})
)
pred = brew.softmax(model, fc8, "pred")
xent = model.LabelCrossEntropy([pred, "label"], "xent")
if device != 'MKL':
loss = model.AveragedLoss(xent, "loss")
return model, 231
def Inception(order, cudnn_ws, device):
my_arg_scope = {'order': order, 'use_cudnn': True,
'cudnn_exhaustive_search': True,
'ws_nbytes_limit': str(cudnn_ws)}
model = ModelHelper(name="inception", arg_scope=my_arg_scope)
conv1 = brew.conv(
model,
"data",
"conv1",
3,
64,
7,
('XavierFill', {}),
('ConstantFill', {}),
stride=2,
pad=3
)
relu1 = brew.relu(model, conv1, "conv1")
pool1 = brew.max_pool(model, relu1, "pool1", kernel=3, stride=2, pad=1)
conv2a = brew.conv(
model, pool1, "conv2a", 64, 64, 1,
('XavierFill', {}), ('ConstantFill', {})
)
conv2a = brew.relu(model, conv2a, conv2a)
conv2 = brew.conv(
model,
conv2a,
"conv2",
64,
192,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1
)
relu2 = brew.relu(model, conv2, "conv2")
pool2 = brew.max_pool(model, relu2, "pool2", kernel=3, stride=2, pad=1)
# Inception modules
inc3 = _InceptionModule(
model, pool2, 192, "inc3", 64, [96, 128], [16, 32], 32
)
inc4 = _InceptionModule(
model, inc3, 256, "inc4", 128, [128, 192], [32, 96], 64
)
pool5 = brew.max_pool(model, inc4, "pool5", kernel=3, stride=2, pad=1)
inc5 = _InceptionModule(
model, pool5, 480, "inc5", 192, [96, 208], [16, 48], 64
)
inc6 = _InceptionModule(
model, inc5, 512, "inc6", 160, [112, 224], [24, 64], 64
)
inc7 = _InceptionModule(
model, inc6, 512, "inc7", 128, [128, 256], [24, 64], 64
)
inc8 = _InceptionModule(
model, inc7, 512, "inc8", 112, [144, 288], [32, 64], 64
)
inc9 = _InceptionModule(
model, inc8, 528, "inc9", 256, [160, 320], [32, 128], 128
)
pool9 = brew.max_pool(model, inc9, "pool9", kernel=3, stride=2, pad=1)
inc10 = _InceptionModule(
model, pool9, 832, "inc10", 256, [160, 320], [32, 128], 128
)
inc11 = _InceptionModule(
model, inc10, 832, "inc11", 384, [192, 384], [48, 128], 128
)
pool11 = brew.average_pool(model, inc11, "pool11", kernel=7, stride=1)
fc = brew.fc(
model, pool11, "fc", 1024, 1000,
('XavierFill', {}), ('ConstantFill', {})
)
# It seems that Soumith's benchmark does not have softmax on top
# for Inception. We will add it anyway so we can have a proper
# backward pass.
pred = brew.softmax(model, fc, "pred")
xent = model.LabelCrossEntropy([pred, "label"], "xent")
if device != 'MKL':
loss = model.AveragedLoss(xent, "loss")
return model, 224
def VGGA(order, cudnn_ws, device):
my_arg_scope = {'order': order, 'use_cudnn': True,
'cudnn_exhaustive_search': True,
'ws_nbytes_limit': str(cudnn_ws)}
model = ModelHelper(name='vgg-a', arg_scope=my_arg_scope)
conv1 = brew.conv(
model,
"data",
"conv1",
3,
64,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1
)
relu1 = brew.relu(model, conv1, "conv1")
pool1 = brew.max_pool(model, relu1, "pool1", kernel=2, stride=2)
conv2 = brew.conv(
model,
pool1,
"conv2",
64,
128,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1
)
relu2 = brew.relu(model, conv2, "conv2")
pool2 = brew.max_pool(model, relu2, "pool2", kernel=2, stride=2)
conv3 = brew.conv(
model,
pool2,
"conv3",
128,
256,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1
)
relu3 = brew.relu(model, conv3, "conv3")
conv4 = brew.conv(
model,
relu3,
"conv4",
256,
256,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1
)
relu4 = brew.relu(model, conv4, "conv4")
pool4 = brew.max_pool(model, relu4, "pool4", kernel=2, stride=2)
conv5 = brew.conv(
model,
pool4,
"conv5",
256,
512,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1
)
relu5 = brew.relu(model, conv5, "conv5")
conv6 = brew.conv(
model,
relu5,
"conv6",
512,
512,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1
)
relu6 = brew.relu(model, conv6, "conv6")
pool6 = brew.max_pool(model, relu6, "pool6", kernel=2, stride=2)
conv7 = brew.conv(
model,
pool6,
"conv7",
512,
512,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1
)
relu7 = brew.relu(model, conv7, "conv7")
conv8 = brew.conv(
model,
relu7,
"conv8",
512,
512,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1
)
relu8 = brew.relu(model, conv8, "conv8")
pool8 = brew.max_pool(model, relu8, "pool8", kernel=2, stride=2)
fcix = brew.fc(
model, pool8, "fcix", 512 * 7 * 7, 4096, ('XavierFill', {}),
('ConstantFill', {})
)
reluix = brew.relu(model, fcix, "fcix")
fcx = brew.fc(
model, reluix, "fcx", 4096, 4096, ('XavierFill', {}), ('ConstantFill', {})
)
relux = brew.relu(model, fcx, "fcx")
fcxi = brew.fc(
model, relux, "fcxi", 4096, 1000, ('XavierFill', {}), ('ConstantFill', {})
)
pred = brew.softmax(model, fcxi, "pred")
xent = model.LabelCrossEntropy([pred, "label"], "xent")
if device != 'MKL':
loss = model.AveragedLoss(xent, "loss")
return model, 231
def create_resnet50(
model,
data,
num_input_channels,
num_labels,
label=None,
is_test=False,
no_loss=False,
no_bias=0,
conv1_kernel=7,
conv1_stride=2,
final_avg_kernel=7,
):
# conv1 + maxpool
brew.conv(
model,
data,
'conv1',
num_input_channels,
64,
weight_init=("MSRAFill", {}),
kernel=conv1_kernel,
stride=conv1_stride,
pad=3,
no_bias=no_bias
)
brew.spatial_bn(
model,
'conv1',
'conv1_spatbn_relu',
64,
epsilon=1e-3,
momentum=0.1,
is_test=is_test
)
brew.relu(model, 'conv1_spatbn_relu', 'conv1_spatbn_relu')
brew.max_pool(model, 'conv1_spatbn_relu', 'pool1', kernel=3, stride=2)
# Residual blocks...
builder = ResNetBuilder(model, 'pool1', no_bias=no_bias,
is_test=is_test, spatial_bn_mom=0.1)
# conv2_x (ref Table 1 in He et al. (2015))
builder.add_bottleneck(64, 64, 256)
builder.add_bottleneck(256, 64, 256)
builder.add_bottleneck(256, 64, 256)
# conv3_x
builder.add_bottleneck(256, 128, 512, down_sampling=True)
for _ in range(1, 4):
builder.add_bottleneck(512, 128, 512)
# conv4_x
builder.add_bottleneck(512, 256, 1024, down_sampling=True)
for _ in range(1, 6):
builder.add_bottleneck(1024, 256, 1024)
# conv5_x
builder.add_bottleneck(1024, 512, 2048, down_sampling=True)
builder.add_bottleneck(2048, 512, 2048)
builder.add_bottleneck(2048, 512, 2048)
# Final layers
final_avg = brew.average_pool(
model,
builder.prev_blob,
'final_avg',
kernel=final_avg_kernel,
stride=1,
)
# Final dimension of the "image" is reduced to 7x7
last_out = brew.fc(
model, final_avg, 'last_out_L{}'.format(num_labels), 2048, num_labels
)
if no_loss:
return last_out
# If we create model for training, use softmax-with-loss
if (label is not None):
(softmax, loss) = model.SoftmaxWithLoss(
[last_out, label],
["softmax", "loss"],
)
return (softmax, loss)
else:
# For inference, we just return softmax
return brew.softmax(model, last_out, "softmax")
def alexnet():
model = ModelHelper(name="r", arg_scope={"order": "NCHW", "is_test": True})
conv1 = brew.conv(
model,
"data",
"conv1",
3,
64,
11, ('XavierFill', {}), ('ConstantFill', {}),
stride=4,
pad=2
)
relu1 = brew.relu(model, conv1, "conv1")
pool1 = brew.max_pool(model, relu1, "pool1", kernel=3, stride=2, pad=0,
legacy_pad=3)
lrn1 = brew.lrn(
model, pool1, "pool1_lrn", size=5, alpha=1.0e-4, beta=0.75, bias=1.0)
conv2 = brew.conv(
model,
lrn1,
"conv2",
64,
192,
5,
('XavierFill', {}),
('ConstantFill', {}),
pad=2
)
relu2 = brew.relu(model, conv2, "conv2")
pool2 = brew.max_pool(model, relu2, "pool2", kernel=3, stride=2)
lrn2 = brew.lrn(
model, pool2, "pool2_lrn", size=5, alpha=1.0e-4, beta=0.75, bias=1.0)
conv3 = brew.conv(
model,
lrn2,
"conv3",
192,
384,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1
)
relu3 = brew.relu(model, conv3, "conv3")
conv4 = brew.conv(
model,
relu3,
"conv4",
384,
256,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1
)
relu4 = brew.relu(model, conv4, "conv4")
conv5 = brew.conv(
model,
relu4,
"conv5",
256,
256,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1
)
relu5 = brew.relu(model, conv5, "conv5")
pool5 = brew.max_pool(model, relu5, "pool5", kernel=3, stride=2)
fc6 = brew.fc(
model,
pool5, "fc6", 256 * 6 * 6, 4096, ('XavierFill', {}),
('ConstantFill', {})
)
relu6 = brew.relu(model, fc6, "fc6")
fc7 = brew.fc(
model, relu6, "fc7", 4096, 4096, ('XavierFill', {}), ('ConstantFill', {})
)
relu7 = brew.relu(model, fc7, "fc7")
drop7 = brew.dropout(model, relu7, "fc7_dropout", is_test=1, ratio=0.5)
fc8 = brew.fc(
model, drop7, "fc8", 4096, 1000, ('XavierFill', {}), ('ConstantFill', {})
)
relu8 = brew.relu(model, fc8, "fc8")
_ = brew.dropout(model, relu8, "fc8_dropout", is_test=1, ratio=0.5)
return model, [(1, 3, 224, 224)]
def _MiniAlexNetNoDropout(self, order):
# First, AlexNet using the cnn wrapper.
model = model_helper.ModelHelper(name="alexnet")
conv1 = brew.conv(
model,
"data",
"conv1",
3,
16,
11,
("XavierFill", {}),
("ConstantFill", {}),
stride=4,
pad=0
)
relu1 = brew.relu(model, conv1, "relu1")
norm1 = brew.lrn(model, relu1, "norm1", size=5, alpha=0.0001, beta=0.75)
pool1 = brew.max_pool(model, norm1, "pool1", kernel=3, stride=2)
conv2 = brew.group_conv(
model,
pool1,
"conv2",
16,
32,
5,
("XavierFill", {}),
("ConstantFill", {"value": 0.1}),
group=2,
stride=1,
pad=2
)
relu2 = brew.relu(model, conv2, "relu2")
norm2 = brew.lrn(model, relu2, "norm2", size=5, alpha=0.0001, beta=0.75)
pool2 = brew.max_pool(model, norm2, "pool2", kernel=3, stride=2)
conv3 = brew.conv(
model,
pool2,
"conv3",
32,
64,
3,
("XavierFill", {'std': 0.01}),
("ConstantFill", {}),
pad=1
)
relu3 = brew.relu(model, conv3, "relu3")
conv4 = brew.group_conv(
model,
relu3,
"conv4",
64,
64,
3,
("XavierFill", {}),
("ConstantFill", {"value": 0.1}),
group=2,
pad=1
)
relu4 = brew.relu(model, conv4, "relu4")
conv5 = brew.group_conv(
model,
relu4,
"conv5",
64,
32,
3,
("XavierFill", {}),
("ConstantFill", {"value": 0.1}),
group=2,
pad=1
)
relu5 = brew.relu(model, conv5, "relu5")
pool5 = brew.max_pool(model, relu5, "pool5", kernel=3, stride=2)
fc6 = brew.fc(
model, pool5, "fc6", 1152, 1024, ("XavierFill", {}),
("ConstantFill", {"value": 0.1})
)
relu6 = brew.relu(model, fc6, "relu6")
fc7 = brew.fc(
model, relu6, "fc7", 1024, 1024, ("XavierFill", {}),
("ConstantFill", {"value": 0.1})
)
relu7 = brew.relu(model, fc7, "relu7")
fc8 = brew.fc(
model, relu7, "fc8", 1024, 5, ("XavierFill", {}),
("ConstantFill", {"value": 0.0})
)
pred = brew.softmax(model, fc8, "pred")
xent = model.LabelCrossEntropy([pred, "label"], "xent")
loss = model.AveragedLoss([xent], ["loss"])
model.AddGradientOperators([loss])
return model
