def test_caffe2_to_onnx(self): caffe2_net = tempfile.NamedTemporaryFile() caffe2_init_net = tempfile.NamedTemporaryFile() output = tempfile.NamedTemporaryFile() model = ModelHelper(name='caffe2-to-onnx-test') brew.relu(model, ["X"], "Y") caffe2_net.write(model.net.Proto().SerializeToString()) caffe2_net.flush() init_model = ModelHelper(name='caffe2-to-onnx-init-test') init_model.net.GivenTensorFill([], 'X', shape=[2, 2], values=np.zeros((2, 2)).flatten().astype(float)) caffe2_init_net.write(init_model.net.Proto().SerializeToString()) caffe2_init_net.flush() result = self._run_command( caffe2_to_onnx, [ caffe2_net.name, '--caffe2-init-net', caffe2_init_net.name, '--output', output.name, ], catch_exceptions=False, ) onnx_model = ModelProto() onnx_model.ParseFromString(output.read()) self.assertEqual(len(onnx_model.graph.node), 1) self.assertEqual(onnx_model.graph.node[0].op_type, 'Relu') self.assertEqual(len(onnx_model.graph.initializer), 1) self.assertEqual(onnx_model.graph.initializer[0].name, onnx_model.graph.input[0].name)
def AddLeNetModel(model, data): conv1 = brew.conv(model, data, 'conv1', dim_in=1, dim_out=32, kernel=3, pad=1) conv1 = brew.relu(model, conv1, conv1) conv2 = brew.conv(model, conv1, 'conv2', dim_in=32, dim_out=128, kernel=3, pad=1) conv2 = brew.relu(model, conv2, conv2) conv3 = brew.conv(model, conv2, 'conv3', dim_in=128, dim_out=256, kernel=3, pad=1) conv3 = brew.relu(model, conv3, conv3) fc3 = brew.fc(model, conv3, 'fc3', dim_in=256 * 28 * 28, dim_out=512) fc3 = brew.relu(model, fc3, fc3) pred = brew.fc(model, fc3, 'pred', 512, 10) softmax = brew.softmax(model, pred, 'softmax') return softmax
def test_caffe2_to_onnx_value_info(self): caffe2_net = tempfile.NamedTemporaryFile() output = tempfile.NamedTemporaryFile() model = ModelHelper(name='caffe2-to-onnx-test') brew.relu(model, ["X"], "Y") caffe2_net.write(model.net.Proto().SerializeToString()) caffe2_net.flush() args = [caffe2_net.name, '--output', output.name] self.assertRaisesRegexp(Exception, 'value info', self._run_command, caffe2_to_onnx, args) args.extend([ '--value-info', json.dumps({ 'X': (TensorProto.FLOAT, (2, 2)), })]) result = self._run_command(caffe2_to_onnx, args) onnx_model = ModelProto() onnx_model.ParseFromString(output.read()) self.assertEqual(len(onnx_model.graph.node), 1) self.assertEqual(onnx_model.graph.node[0].op_type, 'Relu') self.assertEqual(len(onnx_model.graph.initializer), 0)
def add_layers(model, data): # Image size: 32 x 32 x 3 -> 30 x 30 x 60 conv1 = brew.conv(model, data, 'conv1', dim_in=3, dim_out=64, kernel=3) #add relu conv1 = brew.relu(model, conv1, conv1) # Image size: 30 x 30 -> 15 x 15 pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2) # Image size: 15 x 15 -> 12 x 12 conv2 = brew.conv(model, pool1, 'conv2', dim_in=64, dim_out=128, kernel=3) #add relu conv2 = brew.relu(model, conv2, conv2) # Image size: 12 x 12 -> 6 x 6 pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2) # 32 * 6 * 6 stands for dim_out from previous layer multiplied by the image size # Hidden layer fc3 = brew.fc(model, pool2, 'fc3', dim_in=128 * 6 * 6, dim_out=512) #add relu fc3 = brew.relu(model, fc3, fc3) #CO = model.param_init_net.ConstantFill([], "CO", shape=[1], value=0.18) #fc3 = brew.prelu(model, fc3, fc3) #output layer pred = brew.fc(model, fc3, 'pred', 512, 10) #add softmax probs = brew.softmax(model, pred, 'probs') return probs
def test_caffe2_to_onnx_value_info(self): caffe2_net = tempfile.NamedTemporaryFile() output = tempfile.NamedTemporaryFile() model = ModelHelper(name='caffe2-to-onnx-test') brew.relu(model, ["X"], "Y") caffe2_net.write(model.net.Proto().SerializeToString()) caffe2_net.flush() args = [caffe2_net.name, '--output', output.name] self.assertRaisesRegexp(Exception, 'value info', self._run_command, caffe2_to_onnx, args) args.extend( ['--value-info', json.dumps({ 'X': (TensorProto.FLOAT, (2, 2)), })]) result = self._run_command(caffe2_to_onnx, args) onnx_model = ModelProto() onnx_model.ParseFromString(output.read()) self.assertEqual(len(onnx_model.graph.node), 1) self.assertEqual(onnx_model.graph.node[0].op_type, 'Relu') self.assertEqual(len(onnx_model.graph.initializer), 0)
def test_caffe2_to_onnx(self): caffe2_net = tempfile.NamedTemporaryFile() caffe2_init_net = tempfile.NamedTemporaryFile() output = tempfile.NamedTemporaryFile() model = ModelHelper(name='caffe2-to-onnx-test') brew.relu(model, ["X"], "Y") caffe2_net.write(model.net.Proto().SerializeToString()) caffe2_net.flush() init_model = ModelHelper(name='caffe2-to-onnx-init-test') init_model.net.GivenTensorFill([], 'X', shape=[2, 2], values=np.zeros((2, 2)).flatten().astype(float)) caffe2_init_net.write(init_model.net.Proto().SerializeToString()) caffe2_init_net.flush() result = self._run_command( caffe2_to_onnx, [ caffe2_net.name, '--caffe2-init-net', caffe2_init_net.name, '--output', output.name, ]) onnx_model = onnx_pb2.ModelProto() onnx_model.ParseFromString(output.read()) self.assertEqual(len(onnx_model.graph.node), 1) self.assertEqual(onnx_model.graph.node[0].op_type, 'Relu') self.assertEqual(len(onnx_model.graph.initializer), 1) self.assertEqual(onnx_model.graph.initializer[0].name, onnx_model.graph.input[0].name)
def test_fast_memonger_unique_outputs(self): m = model_helper.ModelHelper() fc = [] for i in range(2): z = brew.fc(m, "data{}".format(i), "fc".format(i), dim_in=2, dim_out=2) fc.append(z) r = [] # Trick is here to have same input appear twice in a same Sum for x in fc: for y in fc: r.append(brew.sum(m, [x, y], 1)) concated = brew.concat(m, r, "concated") brew.relu(m, concated, "merged") static_blobs = \ [o for op in m.param_init_net.Proto().op for o in op.output] + \ ["merged"] + ["data{}".format(i) for i in range(len(fc))] optimized_net = memonger.optimize_inference_fast( m.Proto(), static_blobs) for op in optimized_net.op: self.assertEqual(len(op.output), len(set(op.output)), str(op))
def conv_dw(model, inputs, inp, oup, stride, name): W_dw = model.create_param(param_name=name + '_dw_w', shape=[inp, 1, 3, 3], initializer=initializers.update_initializer( None, None, ("XavierFill", {})), tags=ParameterTags.WEIGHT) h = inputs.Conv([W_dw], [name + '_dw'], kernel_h=3, kernel_w=3, stride_h=stride, stride_w=stride, pad_b=1, pad_l=1, pad_r=1, pad_t=1, order='NCHW', group=inp) h = brew.spatial_bn(model, h, name + '_dw_bn', inp, is_test=True) h = brew.relu(model, h, h) h = brew.conv(model, h, name + '_sep', dim_in=inp, dim_out=oup, kernel=1, pad=0, stride=1, no_bias=True) h = brew.spatial_bn(model, h, name + '_sep_bn', oup, is_test=True) h = brew.relu(model, h, h) return h
def AddNetModel_5(model, data): # 0 params, dims [1, 70, 182] input1 = data # 18 params, dims [2, 68, 180] conv2 = brew.conv(model, input1, 'conv2', dim_in=1, dim_out=2, kernel=3, stride=1) # 0 params, dims [2, 34, 90] pool3 = brew.max_pool(model, conv2, 'pool3', kernel=2, stride=2) # 72 params, dims [4, 32, 88] conv4 = brew.conv(model, pool3, 'conv4', dim_in=2, dim_out=4, kernel=3, stride=1) # 0 params, dims [4, 16, 44] pool5 = brew.max_pool(model, conv4, 'pool5', kernel=2, stride=2) # 288 params, dims [8, 14, 42] conv6 = brew.conv(model, pool5, 'conv6', dim_in=4, dim_out=8, kernel=3, stride=1) # 0 params, dims [8, 14, 42] relu7 = brew.relu(model, conv6, 'relu7') # 64 params, dims [2, 7, 21] conv8 = brew.conv(model, relu7, 'conv8', dim_in=8, dim_out=2, kernel=2, stride=2) # 0 params, dims [2, 7, 21] relu9 = brew.relu(model, conv8, 'relu9') # 784 params, dims [8, 1, 3] conv10 = brew.conv(model, relu9, 'conv10', dim_in=2, dim_out=8, kernel=7, stride=7) # 0 params, dims [8, 1, 3] relu11 = brew.relu(model, conv10, 'relu11') # 192 params, dims [8] fc12 = brew.fc(model, relu11, 'fc12', dim_in=24, dim_out=8) # 0 params, dims [8] relu13 = brew.relu(model, fc12, 'relu13') # 16 params, dims [2] output = brew.fc(model, relu13, 'output', dim_in=8, dim_out=2) return output
def simple_mlp(): model = ModelHelper(name="r") brew.relu( model, brew.fc(model, brew.relu(model, brew.fc(model, "data", "fc1", 10, 10), "rl1"), "fc2", 10, 10), "rl2") return model, [(1, 10)]
def AddLeNetModel(model, data): fc1 = brew.fc(model, data, 'fc1', dim_in=784, dim_out=1000) fc1 = brew.relu(model, fc1, fc1) fc2 = brew.fc(model, fc1, 'fc2', dim_in=1000, dim_out=1000) fc2 = brew.relu(model, fc2, fc2) pred = brew.fc(model, fc2, 'fc3', dim_in=1000, dim_out=10) softmax = brew.softmax(model, pred, 'softmax') return softmax
def Add_Original_CIFAR10_Model(model, data, num_classes, image_height, image_width, image_channels, save_png=True): conv1 = brew.conv(model, data, 'conv1', dim_in=image_channels, dim_out=32, kernel=5, stride=1, pad=2) h, w = update_dims(height=image_height, width=image_width, kernel=5, stride=1, pad=2) pool1 = brew.max_pool(model, conv1, 'pool1', kernel=3, stride=2) h, w = update_dims(height=h, width=w, kernel=3, stride=2, pad=0) relu1 = brew.relu(model, pool1, 'relu1') conv2 = brew.conv(model, relu1, 'conv2', dim_in=32, dim_out=32, kernel=5, stride=1, pad=2) h, w = update_dims(height=h, width=w, kernel=5, stride=1, pad=2) relu2 = brew.relu(model, conv2, 'relu2') pool2 = brew.average_pool(model, relu2, 'pool2', kernel=3, stride=2) h, w = update_dims(height=h, width=w, kernel=3, stride=2, pad=0) conv3 = brew.conv(model, pool2, 'conv3', dim_in=32, dim_out=64, kernel=5, stride=1, pad=2) h, w = update_dims(height=h, width=w, kernel=5, stride=1, pad=2) relu3 = brew.relu(model, conv3, 'relu3') pool3 = brew.average_pool(model, relu3, 'pool3', kernel=3, stride=2) h, w = update_dims(height=h, width=w, kernel=3, stride=2, pad=0) fc1 = brew.fc(model, pool3, 'fc1', dim_in=64 * h * w, dim_out=64) fc2 = brew.fc(model, fc1, 'fc2', dim_in=64, dim_out=num_classes) softmax = brew.softmax(model, fc2, 'softmax') if save_png: graph = net_drawer.GetPydotGraph(model.net, rankdir="LR") graph.write_png("CIFAR10_Model.png") return softmax
def test_mobile_exporter(self): model = ModelHelper(name="mobile_exporter_test_model") # Test LeNet brew.conv(model, 'data', 'conv1', dim_in=1, dim_out=20, kernel=5) brew.max_pool(model, 'conv1', 'pool1', kernel=2, stride=2) brew.conv(model, 'pool1', 'conv2', dim_in=20, dim_out=50, kernel=5) brew.max_pool(model, 'conv2', 'pool2', kernel=2, stride=2) brew.fc(model, 'pool2', 'fc3', dim_in=50 * 4 * 4, dim_out=500) brew.relu(model, 'fc3', 'fc3') brew.fc(model, 'fc3', 'pred', 500, 10) brew.softmax(model, 'pred', 'out') # Create our mobile exportable networks workspace.RunNetOnce(model.param_init_net) init_net, predict_net = mobile_exporter.Export( workspace, model.net, model.params ) # Populate the workspace with data np_data = np.random.rand(1, 1, 28, 28).astype(np.float32) workspace.FeedBlob("data", np_data) workspace.CreateNet(model.net) workspace.RunNet(model.net) ref_out = workspace.FetchBlob("out") # Clear the workspace workspace.ResetWorkspace() # Populate the workspace with data workspace.RunNetOnce(init_net) # Fake "data" is populated by init_net, we have to replace it workspace.FeedBlob("data", np_data) # Overwrite the old net workspace.CreateNet(predict_net, True) workspace.RunNet(predict_net.name) manual_run_out = workspace.FetchBlob("out") np.testing.assert_allclose( ref_out, manual_run_out, atol=1e-10, rtol=1e-10 ) # Clear the workspace workspace.ResetWorkspace() # Predictor interface test (simulates writing to disk) predictor = workspace.Predictor( init_net.SerializeToString(), predict_net.SerializeToString() ) # Output is a vector of outputs but we only care about the first and only result predictor_out = predictor.run([np_data]) assert len(predictor_out) == 1 predictor_out = predictor_out[0] np.testing.assert_allclose( ref_out, predictor_out, atol=1e-10, rtol=1e-10 )
def testShapeInferenceConvNet(self): model = model_helper.ModelHelper(name="convtest") model.NHWC2NCHW("data", "data_nchw") brew.conv(model, "data_nchw", 'conv1', 3, 64, weight_init=("MSRAFill", {}), kernel=7, stride=2, pad=3, no_bias=0) brew.spatial_bn(model, 'conv1', 'conv1_spatbn_relu', 64, epsilon=1e-3, is_test=False) brew.relu(model, 'conv1_spatbn_relu', 'conv1_spatbn_relu') brew.max_pool(model, 'conv1_spatbn_relu', 'pool1', kernel=3, stride=2) brew.fc(model, 'pool1', 'fc', dim_in=(64 * 56 * 56), dim_out=100) brew.dropout(model, 'fc', 'fc_drop', is_test=False) model.Sigmoid('fc_drop', 'fc_sigm') brew.softmax(model, 'fc_sigm', 'softmax') model.LabelCrossEntropy(['softmax', 'label'], 'xent') loss = model.AveragedLoss('xent', 'loss') model.AddGradientOperators([loss]) LR = model.param_init_net.ConstantFill([], 'LR', shape=[1], value=0.1) for param in model.GetParams(): param_grad = model.param_to_grad[param] param_momentum = model.param_init_net.ConstantFill([param], param + '_momentum', value=0.0) model.net.MomentumSGDUpdate( [param_grad, param_momentum, LR, param], [param_grad, param_momentum, param], ) workspace.FeedBlob( "data", np.random.rand(16, 227, 227, 3).astype(np.float32), ) workspace.FeedBlob( "label", (100 * np.random.rand(16)).astype(np.int32), ) workspace.FeedBlob( "label", (100 * np.random.rand(16)).astype(np.int32), ) # Then do automatic comparison test: run the next once to # initialize everything self.InferTensorRunAndCompare(model)
def test_mobile_exporter(self): model = ModelHelper(name="mobile_exporter_test_model") # Test LeNet brew.conv(model, 'data', 'conv1', dim_in=1, dim_out=20, kernel=5) brew.max_pool(model, 'conv1', 'pool1', kernel=2, stride=2) brew.conv(model, 'pool1', 'conv2', dim_in=20, dim_out=50, kernel=5) brew.max_pool(model, 'conv2', 'pool2', kernel=2, stride=2) brew.fc(model, 'pool2', 'fc3', dim_in=50 * 4 * 4, dim_out=500) brew.relu(model, 'fc3', 'fc3') brew.fc(model, 'fc3', 'pred', 500, 10) brew.softmax(model, 'pred', 'out') # Create our mobile exportable networks workspace.RunNetOnce(model.param_init_net) init_net, predict_net = mobile_exporter.Export(workspace, model.net, model.params) # Populate the workspace with data np_data = np.random.rand(1, 1, 28, 28).astype(np.float32) workspace.FeedBlob("data", np_data) workspace.CreateNet(model.net) workspace.RunNet(model.net) ref_out = workspace.FetchBlob("out") # Clear the workspace workspace.ResetWorkspace() # Populate the workspace with data workspace.RunNetOnce(init_net) # Fake "data" is populated by init_net, we have to replace it workspace.FeedBlob("data", np_data) # Overwrite the old net workspace.CreateNet(predict_net, True) workspace.RunNet(predict_net.name) manual_run_out = workspace.FetchBlob("out") np.testing.assert_allclose(ref_out, manual_run_out, atol=1e-10, rtol=1e-10) # Clear the workspace workspace.ResetWorkspace() # Predictor interface test (simulates writing to disk) predictor = workspace.Predictor(init_net.SerializeToString(), predict_net.SerializeToString()) # Output is a vector of outputs but we only care about the first and only result predictor_out = predictor.run([np_data]) assert len(predictor_out) == 1 predictor_out = predictor_out[0] np.testing.assert_allclose(ref_out, predictor_out, atol=1e-10, rtol=1e-10)
def AddLeNetModel(model, data, device_opts): ''' This part is the standard LeNet model: from data to the softmax prediction. For each convolutional layer we specify dim_in - number of input channels and dim_out - number or output channels. Also each Conv and MaxPool layer changes the image size. For example, kernel of size 5 reduces each side of an image by 4. While when we have kernel and stride sizes equal 2 in a MaxPool layer, it divides each side in half. ''' with core.DeviceScope(device_opts): ############# Change dim_in value if images are more than 1 color channel # Image size: 64x64 conv1 = brew.conv(model, data, 'conv1', dim_in=1, dim_out=32, kernel=5) h, w = update_dims(height=image_height, width=image_width, kernel=5, stride=1, pad=0) # Image size: 60x60 pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2) h, w = update_dims(height=h, width=w, kernel=2, stride=2, pad=0) relu1 = brew.relu(model, pool1, 'relu1') # Image size: 30x30 conv2 = brew.conv(model, relu1, 'conv2', dim_in=32, dim_out=64, kernel=5) h, w = update_dims(height=h, width=w, kernel=5, stride=1, pad=0) # Image size: 26x26 pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2) h, w = update_dims(height=h, width=w, kernel=2, stride=2, pad=0) relu2 = brew.relu(model, pool2, 'relu2') # Image size: 13x13 conv3 = brew.conv(model, relu2, 'conv3', dim_in=64, dim_out=64, kernel=5) h, w = update_dims(height=h, width=w, kernel=5, stride=1, pad=0) # Image size: 9x9 pool3 = brew.max_pool(model, conv3, 'pool3', kernel=2, stride=2) h, w = update_dims(height=h, width=w, kernel=2, stride=2, pad=0) relu3 = brew.relu(model, pool3, 'relu3') # Image size: 4x4 # 50 * 4 * 4 stands for dim_out from previous layer multiplied by the image size ############# Change dim_in value if images are not 28x28 fc3 = brew.fc(model, relu3, 'fc3', dim_in=64 * h * w, dim_out=500) fc3 = brew.relu(model, fc3, fc3) pred = brew.fc(model, fc3, 'pred', 500, num_classes) softmax = brew.softmax(model, pred, 'softmax') return softmax
def simple_cnn(): model = ModelHelper(name="r", arg_scope={"order": "NCHW", "is_test": True}) brew.conv( model, "data", 'conv1', 3, 16, kernel=3, stride=1 ) brew.spatial_bn( model, 'conv1', 'conv1_spatbn', 16, epsilon=1e-3 ) brew.relu(model, 'conv1_spatbn', 'relu1') return model, (1, 3, 32, 32)
def create_resnet( model, data, num_input_channels, num_groups, num_labels, device_opts, is_test=False, ): with core.DeviceScope(device_opts): filters = [16, 32, 64] brew.conv(model, data, 'conv1', num_input_channels, filters[0], no_bias=1, kernel=3, stride=1, pad=1) builder = ResNetBuilder(model, 'conv1', no_bias=1, is_test=is_test) # input: 32x32x16 output: 32x32x16 for _ in range(num_groups): builder.add_simple_block(filters[0], filters[0], down_sampling=False) # input: 32x32x16 output: 16x16x32 builder.add_simple_block(filters[0], filters[1], down_sampling=True) for _ in range(1, num_groups): builder.add_simple_block(filters[1], filters[1], down_sampling=False) # input: 16x16x32 output: 8x8x64 builder.add_simple_block(filters[1], filters[2], down_sampling=True) for _ in range(1, num_groups): builder.add_simple_block(filters[2], filters[2], down_sampling=False) brew.spatial_bn(model, builder.prev_blob, 'last_spatbn', filters[2], epsilon=1e-3, is_test=is_test) brew.relu(model, 'last_spatbn', 'last_relu') # Final layers brew.average_pool(model, 'last_relu', 'final_avg', kernel=8, stride=1) last_out = brew.fc(model, 'final_avg', 'last_out', 64, num_labels) return last_out
def AddMLP(model, data, batch_size): ''' Implement MLP model on MNIST ''' num_units = 4096 # number of nuerons in fc layer num_labels = 10 # for 10 classes in mnist drop1 = brew.dropout(model, data, 'drop1', ratio=0.5, is_test=0) fc2 = brew.fc(model, drop1, 'fc2', dim_in=1 * 28 * 28, dim_out=num_units) model.Reshape([fc2], [fc2, 'fc2_old_shape'], shape=(batch_size, num_units, 1, 1)) bn2 = brew.spatial_bn(model, fc2, 'bn2', num_units, epsilon=1e-4, momentum=0.9) relu2 = brew.relu(model, bn2, 'relu2') fc3 = brew.fc(model, relu2, 'fc3', dim_in=num_units, dim_out=num_units) model.Reshape([fc3], [fc3, 'fc3_old_shape'], shape=(batch_size, num_units, 1, 1)) bn3 = brew.spatial_bn(model, fc3, 'bn3', num_units, epsilon=1e-4, momentum=0.9) relu3 = brew.relu(model, bn3, 'relu3') fc4 = brew.fc(model, relu3, 'fc4', dim_in=num_units, dim_out=num_units) model.Reshape([fc4], [fc4, 'fc4_old_shape'], shape=(batch_size, num_units, 1, 1)) bn4 = brew.spatial_bn(model, fc4, 'bn4', num_units, epsilon=1e-4, momentum=0.9) relu4 = brew.relu(model, bn4, 'relu4') fc5 = brew.fc(model, relu4, 'fc5', dim_in=num_units, dim_out=num_labels) model.Reshape([fc5], [fc5, 'fc5_old_shape'], shape=(batch_size, num_labels, 1, 1)) bn5 = brew.spatial_bn(model, fc5, 'bn5', num_labels, epsilon=1e-4, momentum=0.9) softmax = brew.softmax(model, bn5, 'softmax') return softmax
def AddUpgradedLeNetModel_GPU(model, data, num_classes, image_height, image_width, image_channels, device_opts): with core.DeviceScope(device_opts): # Image size: 64x64 conv1 = brew.conv(model, data, 'conv1', dim_in=image_channels, dim_out=32, kernel=5) h, w = update_dims(height=image_height, width=image_width, kernel=5, stride=1, pad=0) # Image size: 60x60 pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2) h, w = update_dims(height=h, width=w, kernel=2, stride=2, pad=0) relu1 = brew.relu(model, pool1, 'relu1') # Image size: 30x30 conv2 = brew.conv(model, relu1, 'conv2', dim_in=32, dim_out=64, kernel=5) h, w = update_dims(height=h, width=w, kernel=5, stride=1, pad=0) # Image size: 26x26 pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2) h, w = update_dims(height=h, width=w, kernel=2, stride=2, pad=0) relu2 = brew.relu(model, pool2, 'relu2') # Image size: 13x13 conv3 = brew.conv(model, relu2, 'conv3', dim_in=64, dim_out=64, kernel=5) h, w = update_dims(height=h, width=w, kernel=5, stride=1, pad=0) # Image size: 9x9 pool3 = brew.max_pool(model, conv3, 'pool3', kernel=2, stride=2) h, w = update_dims(height=h, width=w, kernel=2, stride=2, pad=0) relu3 = brew.relu(model, pool3, 'relu3') # Image size: 4x4 # 50 * 4 * 4 stands for dim_out from previous layer multiplied by the image size ############# Change dim_in value if images are not 28x28 fc3 = brew.fc(model, relu3, 'fc3', dim_in=64 * h * w, dim_out=500) fc3 = brew.relu(model, fc3, fc3) pred = brew.fc(model, fc3, 'pred', 500, num_classes) softmax = brew.softmax(model, pred, 'softmax') return softmax
def AddLeNetModel(model, data): ''' This part is the standard LeNet model: from data to the softmax prediction. For each convolutional layer we specify dim_in - number of input channels and dim_out - number or output channels. Also each Conv and MaxPool layer changes the image size. For example, kernel of size 5 reduces each side of an image by 4. While when we have kernel and stride sizes equal 2 in a MaxPool layer, it divides each side in half. ''' # Image size: 28 x 28 -> 24 x 24 conv1 = brew.conv(model, data, 'conv1', dim_in=1, dim_out=20, kernel=5) # Image size: 24 x 24 -> 12 x 12 pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2) # Image size: 12 x 12 -> 8 x 8 conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=50, kernel=5) # Image size: 8 x 8 -> 4 x 4 pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2) # 50 * 4 * 4 stands for dim_out from previous layer multiplied by the image size fc3 = brew.fc(model, pool2, 'fc3', dim_in=50 * 4 * 4, dim_out=500) fc3 = brew.relu(model, fc3, fc3) pred = brew.fc(model, fc3, 'pred', 500, 10) softmax = brew.softmax(model, pred, 'softmax') return softmax
def AddLeNetModel(model, data, save_png=True): """This part is the standard LeNet model: from data to the softmax prediction. For each convolutional layer we specify dim_in - number of input channels and dim_out - number or output channels. Also each Conv and MaxPool layer changes the image size. For example, kernel of size 5 reduces each side of an image by 4. While when we have kernel and stride sizes equal 2 in a MaxPool layer, it divides each side in half. """ conv1 = brew.conv(model, data, 'conv1', dim_in=1, dim_out=20, kernel=5) pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2) conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=50, kernel=5) pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2) fc3 = brew.fc(model, pool2, 'fc3', dim_in=50 * 4 * 4, dim_out=500) relu3 = brew.relu(model, fc3, 'relu3') pred = brew.fc(model, relu3, 'pred', dim_in=500, dim_out=10) softmax = brew.softmax(model, pred, 'softmax') if save_png: graph = net_drawer.GetPydotGraph(model.net) graph.write_png("LeNet.png") return softmax
def meta_conv( model, inputs, dim_in=64, dim_out=64, kernel=3, pad=1, stride=1, module_seq=None, conv_seq=None, ): # convolution layer conv = brew.conv(model, inputs, 'conv{}_{}'.format(module_seq, conv_seq), dim_in=dim_in, dim_out=dim_out, kernel=kernel, stride=stride, pad=pad) # ReLU layer relu = brew.relu( model, conv, conv # in-place ) return relu
def add_lenet_conv_topk(model, data, **kwargs): ''' Based on original LeNet structure, according to meProp, we add topk gradient selecting layer for conv layers ''' # resolving arguments config = kwargs.get('config', {}) k = kwargs.get('k', 10) # Image size: 28 x 28 -> 24 x 24 conv1 = brew.conv(model, data, 'conv1', dim_in=3, dim_out=20, kernel=5) conv1_hook = model.net.TopKGradHook(conv1, 'conv1_hook', k=k) # Image size: 24 x 24 -> 12 x 12 pool1 = brew.max_pool(model, conv1_hook, 'pool1', kernel=2, stride=2) # Image size: 12 x 12 -> 8 x 8 conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=50, kernel=5) conv2_hook = model.net.TopKGradHook(conv2, 'conv2_hook', k=k) # Image size: 8 x 8 -> 4 x 4 pool2 = brew.max_pool(model, conv2_hook, 'pool2', kernel=2, stride=2) # 50 * 4 * 4 stands for dim_out from previous layer multiplied by the image size fc3 = brew.fc(model, pool2, 'fc3', dim_in=50 * 4 * 4, dim_out=500) relu = brew.relu(model, fc3, fc3) pred = brew.fc(model, relu, 'pred', 500, config['model_arch']['num_classes']) softmax = brew.softmax(model, pred, 'softmax') return softmax
def add_lenet_fc_topk(model, data, **kwargs): ''' Based on original LeNet structure, according to meProp, we add topk gradient selecting layer for fc layers ''' # resolving arguments k = kwargs.get('k', 10) # Image size: 28 x 28 -> 24 x 24 conv1 = brew.conv(model, data, 'conv1', dim_in=3, dim_out=20, kernel=5) # Image size: 24 x 24 -> 12 x 12 pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2) # Image size: 12 x 12 -> 8 x 8 conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=50, kernel=5) # Image size: 8 x 8 -> 4 x 4 pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2) # DEBUG PRINT model.net.Print(pool2, []) # set topkgradhook for the hidden fc layers fc3 = brew.fc(model, pool2, 'fc3', dim_in=50 * 4 * 4, dim_out=500) fc3_hook = model.net.TopKGradHook(fc3, 'fc3_hook', k=k) relu = brew.relu(model, fc3_hook, fc3_hook) # don't set topkgradhook for the output fc layer pred = brew.fc(model, relu, 'pred', 500, 10) softmax = brew.softmax(model, pred, 'softmax') return softmax
def add_relu(self): self.prev_blob = brew.relu( self.model, self.prev_blob, self.prev_blob, # in-place ) return self.prev_blob
def AddNetModel_3(model, data): # 182x70x1 -> 90x34x3 conv1 = brew.conv(model, data, 'conv1', dim_in=1, dim_out=6, kernel=3) pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2) # 90x34x3 -> 44x16x5 conv2 = brew.conv(model, pool1, 'conv2', dim_in=6, dim_out=8, kernel=3) pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2) # 44x16x5 -> 21x7x7 conv3 = brew.conv(model, pool2, 'conv3', dim_in=8, dim_out=10, kernel=3) pool3 = brew.max_pool(model, conv3, 'pool3', kernel=2, stride=2) relu4 = brew.relu(model, pool3, 'relu4') # 21x7x7 -> 2 fc5 = brew.fc(model, relu4, 'fc5', dim_in=21 * 7 * 10, dim_out=32) relu5 = brew.relu(model, fc5, 'relu5') output = brew.fc(model, relu5, 'output', dim_in=32, dim_out=2) return output
def AddLeNetModel(model, data): ''' This part is the standard LeNet model: from data to the softmax prediction. For each convolutional layer we specify dim_in - number of input channels and dim_out - number or output channels. Also each Conv and MaxPool layer changes the image size. For example, kernel of size 5 reduces each side of an image by 4. While when we have kernel and stride sizes equal 2 in a MaxPool layer, it divides each side in half. Para encadenar layers, la dimension de salida tiene que coincidir con a dimension de entrada de la siguiente layer (sin contar las pool, solo las conv) ''' #Each layer is created using brew, which in turn is using its operator hooks to instantiate each Op. # Image size: 28 x 28 -> 24 x 24 conv1 = brew.conv(model, data, 'conv1', dim_in=1, dim_out=20, kernel=5) # Image size: 24 x 24 -> 12 x 12 pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2) # Image size: 12 x 12 -> 8 x 8 conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=100, kernel=5) # Image size: 8 x 8 -> 4 x 4 pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2) # 100 * 4 * 4 stands for dim_out from previous layer multiplied by the image size fc3 = brew.fc(model, pool2, 'fc3', dim_in=100 * 4 * 4, dim_out=500) #TODO: RELU despues de cada conv layer -> no mejora mucho la accuracy relu = brew.relu(model, fc3, fc3) pred = brew.fc(model, relu, 'pred', 500, 10) softmax = brew.softmax(model, pred, 'softmax') return softmax
def conv_factory(self, model, v, num_in_channels, num_filters, kernel, stride=1, pad=0, relu=True, name='conv'): """Standard convolution block: Conv -> BatchNorm -> Activation """ if isinstance(pad, int): pad_t = pad_b = pad_l = pad_r = pad elif isinstance(pad, list) or isinstance(pad, tuple): if len(pad) == 2: pad_t = pad_b = pad[0] pad_l = pad_r = pad[1] elif len(pad) == 4: pad_t = pad[0] pad_b = pad[1] pad_l = pad[2] pad_r = pad[3] else: assert False, "Invalid length of pad array. Expecting 2 or 4 but have: " + str(pad) else: assert False, "Invalid type of padding: " + str(pad) v = brew.conv(model, v, name + '_conv', num_in_channels, num_filters, kernel=kernel, pad_t=pad_t, pad_l=pad_l, pad_b=pad_b, pad_r=pad_r, stride=stride) v = brew.spatial_bn(model, v, name+'_bn', num_filters, eps=2e-5, momentum=0.9, is_test=(self.phase == 'inference')) if relu is True: v = brew.relu(model, v, name + '_relu') return v
def AddLeNetModel(model, data): ''' This part is the standard LeNet model: from data to the softmax prediction. For each convolutional layer we specify dim_in - number of input channels and dim_out - number or output channels. Also each Conv and MaxPool layer changes the image size. For example, kernel of size 5 reduces each side of an image by 4. While when we have kernel and stride sizes equal 2 in a MaxPool layer, it divides each side in half. ''' # Image size: 28 x 28 -> 24 x 24 / 350 x 350 -> 346 x 346 conv1 = brew.conv(model, data, 'conv1', dim_in=1, dim_out=20, kernel=3) # Image size: 346 x 346 -> 173 x 173 pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2) # Image size: 174 x 174 conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=50, kernel=5) # Image size: 174 x 174 -> 87 x 87 pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2) # 50 * 4 * 4 stands for dim_out from previous layer multiplied by the image size # Here, the data is flattened from a tensor of dimension 50x4x4 to a vector of length 50*4*4 fc3 = brew.fc(model, pool2, 'fc3', dim_in=50 * 87 * 87, dim_out=500) relu3 = brew.relu(model, fc3, 'relu3') # Last FC Layer pred = brew.fc(model, relu3, 'pred', dim_in=500, dim_out=10) # Softmax Layer softmax = brew.softmax(model, pred, 'softmax') return softmax
def AddLeNetModel(model, data, num_classes, image_height, image_width, image_channels): # Image size: 28 x 28 -> 24 x 24 ############# Change dim_in value if images are more than 1 color channel conv1 = brew.conv(model, data, 'conv1', dim_in=1, dim_out=20, kernel=5) h, w = update_dims(height=image_height, width=image_width, kernel=5, stride=1, pad=0) # Image size: 24 x 24 -> 12 x 12 pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2) h, w = update_dims(height=h, width=w, kernel=2, stride=2, pad=0) # Image size: 12 x 12 -> 8 x 8 conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=50, kernel=5) h, w = update_dims(height=h, width=w, kernel=5, stride=1, pad=0) # Image size: 8 x 8 -> 4 x 4 pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2) h, w = update_dims(height=h, width=w, kernel=2, stride=2, pad=0) # 50 * 4 * 4 stands for dim_out from previous layer multiplied by the image size ############# Change dim_in value if images are not 28x28 fc3 = brew.fc(model, pool2, 'fc3', dim_in=50 * h * w, dim_out=500) fc3 = brew.relu(model, fc3, fc3) pred = brew.fc(model, fc3, 'pred', 500, num_classes) softmax = brew.softmax(model, pred, 'softmax') return softmax
def AddNet(model, data): # image size: 28x28 -> 24x24 conv1 = brew.conv(model, data, "conv1", dim_in=1, dim_out=20, kernel=5, stride=1) # image size: 24x24 -> 12x12 pool1 = brew.max_pool(model, conv1, "pool1", kernel=2, stride=2) # image size: 12x12 -> 8x8 conv2 = brew.conv(model, pool1, "conv2", dim_in=20, dim_out=100, kernel=5, stride=1) # image size: 8x8 -> 4x4 pool2 = brew.max_pool(model, conv2, "pool2", kernel=2, stride=2) # image size: 100x4x4 -> 500 fc3 = brew.fc(model, pool2, "fc3", dim_in=100 * 4 * 4, dim_out=500) relu = brew.relu(model, fc3, fc3) # image size: 500 -> 10 fc4 = brew.fc(model, fc3, "fc4", dim_in=500, dim_out=10) # softmax softmax = brew.softmax(model, fc4, "softmax") return softmax
def test_caffe2_simple_model(self): model = ModelHelper(name="mnist") # how come those inputs don't break the forward pass =.=a workspace.FeedBlob("data", np.random.randn(1, 3, 64, 64).astype(np.float32)) workspace.FeedBlob("label", np.random.randn(1, 1000).astype(np.int)) with core.NameScope("conv1"): conv1 = brew.conv(model, "data", 'conv1', dim_in=1, dim_out=20, kernel=5) # Image size: 24 x 24 -> 12 x 12 pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2) # Image size: 12 x 12 -> 8 x 8 conv2 = brew.conv(model, pool1, 'conv2', dim_in=20, dim_out=100, kernel=5) # Image size: 8 x 8 -> 4 x 4 pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2) with core.NameScope("classifier"): # 50 * 4 * 4 stands for dim_out from previous layer multiplied by the image size fc3 = brew.fc(model, pool2, 'fc3', dim_in=100 * 4 * 4, dim_out=500) relu = brew.relu(model, fc3, fc3) pred = brew.fc(model, relu, 'pred', 500, 10) softmax = brew.softmax(model, pred, 'softmax') xent = model.LabelCrossEntropy([softmax, "label"], 'xent') # compute the expected loss loss = model.AveragedLoss(xent, "loss") model.net.RunAllOnMKL() model.param_init_net.RunAllOnMKL() model.AddGradientOperators([loss], skip=1) blob_name_tracker = {} graph = c2_graph.model_to_graph_def( model, blob_name_tracker=blob_name_tracker, shapes={}, show_simplified=False, ) compare_proto(graph, self)
def testShapeInferenceConvNet(self): model = model_helper.ModelHelper(name="convtest") model.NHWC2NCHW("data", "data_nchw") brew.conv(model, "data_nchw", 'conv1', 3, 64, weight_init=("MSRAFill", {}), kernel=7, stride=2, pad=3, no_bias=0) brew.spatial_bn(model, 'conv1', 'conv1_spatbn_relu', 64, epsilon=1e-3, is_test=False) brew.relu(model, 'conv1_spatbn_relu', 'conv1_spatbn_relu') brew.max_pool(model, 'conv1_spatbn_relu', 'pool1', kernel=3, stride=2) brew.fc(model, 'pool1', 'fc', dim_in=(64 * 56 * 56), dim_out=100) brew.dropout(model, 'fc', 'fc_drop', is_test=False) model.Sigmoid('fc_drop', 'fc_sigm') brew.softmax(model, 'fc_sigm', 'softmax') model.LabelCrossEntropy(['softmax', 'label'], 'xent') loss = model.AveragedLoss('xent', 'loss') model.AddGradientOperators([loss]) LR = model.param_init_net.ConstantFill( [], 'LR', shape=[1], value=0.1 ) for param in model.GetParams(): param_grad = model.param_to_grad[param] param_momentum = model.param_init_net.ConstantFill( [param], param + '_momentum', value=0.0 ) model.net.MomentumSGDUpdate( [param_grad, param_momentum, LR, param], [param_grad, param_momentum, param], ) workspace.FeedBlob( "data", np.random.rand(16, 227, 227, 3).astype(np.float32), ) workspace.FeedBlob( "label", (100 * np.random.rand(16)).astype(np.int32), ) workspace.FeedBlob( "label", (100 * np.random.rand(16)).astype(np.int32), ) # Then do automatic comparison test: run the next once to # initialize everything self.InferTensorRunAndCompare(model)
def test_relu(self): Xpos = np.ones((5, 5)).astype(np.float32) - 0.5 Xneg = np.ones((5, 5)).astype(np.float32) - 1.5 workspace.FeedBlob("xpos", Xpos) workspace.FeedBlob("xneg", Xneg) model = ModelHelper(name="test_model") brew.relu(model, "xpos", "out_xpos") brew.relu(model, "xneg", "out_xneg") model.Validate() workspace.RunNetOnce(model.param_init_net) workspace.RunNetOnce(model.net) pos = workspace.FetchBlob("out_xpos") self.assertAlmostEqual(pos.mean(), 0.5) neg = workspace.FetchBlob("out_xneg") self.assertAlmostEqual(neg.mean(), 0)
def simple_mlp(): model = ModelHelper(name="r") brew.relu( model, brew.fc( model, brew.relu( model, brew.fc( model, "data", "fc1", 10, 10), "rl1"), "fc2", 10, 10), "rl2") return model, (1, 10)
def test_fast_memonger_unique_outputs(self): m = model_helper.ModelHelper() fc = [] for i in range(2): z = brew.fc( m, "data{}".format(i), "fc".format(i), dim_in=2, dim_out=2) fc.append(z) r = [] # Trick is here to have same input appear twice in a same Sum for x in fc: for y in fc: r.append(brew.sum(m, [x, y], 1)) concated = brew.concat(m, r, "concated") brew.relu(m, concated, "merged") static_blobs = \ [o for op in m.param_init_net.Proto().op for o in op.output] + \ ["merged"] + ["data{}".format(i) for i in range(len(fc))] optimized_net = memonger.optimize_inference_fast( m.Proto(), static_blobs) for op in optimized_net.op: self.assertEqual(len(op.output), len(set(op.output)), str(op))
def create_resnet_32x32( model, data, num_input_channels, num_groups, num_labels, is_test=False ): ''' Create residual net for smaller images (sec 4.2 of He et. al (2015)) num_groups = 'n' in the paper ''' # conv1 + maxpool brew.conv( model, data, 'conv1', num_input_channels, 16, kernel=3, stride=1 ) brew.spatial_bn( model, 'conv1', 'conv1_spatbn', 16, epsilon=1e-3, is_test=is_test ) brew.relu(model, 'conv1_spatbn', 'relu1') # Number of blocks as described in sec 4.2 filters = [16, 32, 64] builder = ResNetBuilder(model, 'relu1', is_test=is_test) prev_filters = 16 for groupidx in range(0, 3): for blockidx in range(0, 2 * num_groups): builder.add_simple_block( prev_filters if blockidx == 0 else filters[groupidx], filters[groupidx], down_sampling=(True if blockidx == 0 and groupidx > 0 else False)) prev_filters = filters[groupidx] # Final layers brew.average_pool( model, builder.prev_blob, 'final_avg', kernel=8, stride=1 ) brew.fc(model, 'final_avg', 'last_out', 64, num_labels) softmax = brew.softmax(model, 'last_out', 'softmax') return softmax
def test_tensorboard_graphs(self): model = model_helper.ModelHelper(name="overfeat") data, label = brew.image_input( model, ["db"], ["data", "label"], is_test=0 ) with core.NameScope("conv1"): conv1 = brew.conv(model, data, "conv1", 3, 96, 11, stride=4) relu1 = brew.relu(model, conv1, conv1) pool1 = brew.max_pool(model, relu1, "pool1", kernel=2, stride=2) with core.NameScope("classifier"): fc = brew.fc(model, pool1, "fc", 4096, 1000) pred = brew.softmax(model, fc, "pred") xent = model.LabelCrossEntropy([pred, label], "xent") loss = model.AveragedLoss(xent, "loss") model.AddGradientOperators([loss], skip=1) c2_dir = tempfile.mkdtemp() tf_dir = tempfile.mkdtemp() with open(os.path.join(c2_dir, "init"), "w") as f: f.write(str(model.param_init_net.Proto())) with open(os.path.join(c2_dir, "net"), "w") as f: f.write(str(model.net.Proto())) runner = click.testing.CliRunner() result = runner.invoke( tb.cli, ["tensorboard-graphs", "--c2-netdef", os.path.join(c2_dir, "init"), "--c2-netdef", os.path.join(c2_dir, "net"), "--tf-dir", tf_dir]) self.assertEqual(result.exit_code, 0) entries = list(os.walk(tf_dir)) self.assertEqual(len(entries), 1) ((d, _, (fname,)),) = entries self.assertEqual(tf_dir, d) events = list(tf.train.summary_iterator(os.path.join(tf_dir, fname))) self.assertEqual(len(events), 3) events = events[1:] nets = [model.param_init_net, model.net] for i, (event, net) in enumerate(zip(events, nets), start=1): self.assertEqual(event.step, i) self.assertEqual(event.wall_time, i) g = tf.GraphDef() g.ParseFromString(event.graph_def) self.assertMultiLineEqual( str(g), str(tb_exporter.nets_to_graph_def([net])))
def AlexNet(order, cudnn_ws): my_arg_scope = { 'order': order, 'use_cudnn': True, 'cudnn_exhaustive_search': True, } if cudnn_ws: my_arg_scope['ws_nbytes_limit'] = cudnn_ws model = model_helper.ModelHelper( name="alexnet", arg_scope=my_arg_scope, ) conv1 = brew.conv( model, "data", "conv1", 3, 64, 11, ('XavierFill', {}), ('ConstantFill', {}), stride=4, pad=2 ) relu1 = brew.relu(model, conv1, "conv1") pool1 = brew.max_pool(model, relu1, "pool1", kernel=3, stride=2) conv2 = brew.conv( model, pool1, "conv2", 64, 192, 5, ('XavierFill', {}), ('ConstantFill', {}), pad=2 ) relu2 = brew.relu(model, conv2, "conv2") pool2 = brew.max_pool(model, relu2, "pool2", kernel=3, stride=2) conv3 = brew.conv( model, pool2, "conv3", 192, 384, 3, ('XavierFill', {}), ('ConstantFill', {}), pad=1 ) relu3 = brew.relu(model, conv3, "conv3") conv4 = brew.conv( model, relu3, "conv4", 384, 256, 3, ('XavierFill', {}), ('ConstantFill', {}), pad=1 ) relu4 = brew.relu(model, conv4, "conv4") conv5 = brew.conv( model, relu4, "conv5", 256, 256, 3, ('XavierFill', {}), ('ConstantFill', {}), pad=1 ) relu5 = brew.relu(model, conv5, "conv5") pool5 = brew.max_pool(model, relu5, "pool5", kernel=3, stride=2) fc6 = brew.fc( model, pool5, "fc6", 256 * 6 * 6, 4096, ('XavierFill', {}), ('ConstantFill', {}) ) relu6 = brew.relu(model, fc6, "fc6") fc7 = brew.fc( model, relu6, "fc7", 4096, 4096, ('XavierFill', {}), ('ConstantFill', {}) ) relu7 = brew.relu(model, fc7, "fc7") fc8 = brew.fc( model, relu7, "fc8", 4096, 1000, ('XavierFill', {}), ('ConstantFill', {}) ) pred = brew.softmax(model, fc8, "pred") xent = model.net.LabelCrossEntropy([pred, "label"], "xent") model.net.AveragedLoss(xent, "loss") return model, 224
def simple_relu(): model = ModelHelper(name="r") brew.relu(model, "data", "fc") return model, (1, 10)
def _InceptionModule( model, input_blob, input_depth, output_name, conv1_depth, conv3_depths, conv5_depths, pool_depth ): # path 1: 1x1 conv conv1 = brew.conv( model, input_blob, output_name + ":conv1", input_depth, conv1_depth, 1, ('XavierFill', {}), ('ConstantFill', {}) ) conv1 = brew.relu(model, conv1, conv1) # path 2: 1x1 conv + 3x3 conv conv3_reduce = brew.conv( model, input_blob, output_name + ":conv3_reduce", input_depth, conv3_depths[0], 1, ('XavierFill', {}), ('ConstantFill', {}) ) conv3_reduce = brew.relu(model, conv3_reduce, conv3_reduce) conv3 = brew.conv( model, conv3_reduce, output_name + ":conv3", conv3_depths[0], conv3_depths[1], 3, ('XavierFill', {}), ('ConstantFill', {}), pad=1 ) conv3 = brew.relu(model, conv3, conv3) # path 3: 1x1 conv + 5x5 conv conv5_reduce = brew.conv( model, input_blob, output_name + ":conv5_reduce", input_depth, conv5_depths[0], 1, ('XavierFill', {}), ('ConstantFill', {}) ) conv5_reduce = brew.relu(model, conv5_reduce, conv5_reduce) conv5 = brew.conv( model, conv5_reduce, output_name + ":conv5", conv5_depths[0], conv5_depths[1], 5, ('XavierFill', {}), ('ConstantFill', {}), pad=2 ) conv5 = brew.relu(model, conv5, conv5) # path 4: pool + 1x1 conv pool = brew.max_pool( model, input_blob, output_name + ":pool", kernel=3, stride=1, pad=1 ) pool_proj = brew.conv( model, pool, output_name + ":pool_proj", input_depth, pool_depth, 1, ('XavierFill', {}), ('ConstantFill', {}) ) pool_proj = brew.relu(model, pool_proj, pool_proj) output = brew.concat(model, [conv1, conv3, conv5, pool_proj], output_name) return output
def Relu(self, *args, **kwargs): return brew.relu( self, *args, order=self.order, use_cudnn=self.use_cudnn, **kwargs )
def OverFeat(order, cudnn_ws, device): my_arg_scope = {'order': order, 'use_cudnn': True, 'cudnn_exhaustive_search': True, 'ws_nbytes_limit': str(cudnn_ws)} model = ModelHelper(name='overfeat', arg_scope=my_arg_scope) conv1 = brew.conv( model, "data", "conv1", 3, 96, 11, ('XavierFill', {}), ('ConstantFill', {}), stride=4 ) relu1 = brew.relu(model, conv1, "conv1") pool1 = brew.max_pool(model, relu1, "pool1", kernel=2, stride=2) conv2 = brew.conv( model, pool1, "conv2", 96, 256, 5, ('XavierFill', {}), ('ConstantFill', {}) ) relu2 = brew.relu(model, conv2, "conv2") pool2 = brew.max_pool(model, relu2, "pool2", kernel=2, stride=2) conv3 = brew.conv( model, pool2, "conv3", 256, 512, 3, ('XavierFill', {}), ('ConstantFill', {}), pad=1 ) relu3 = brew.relu(model, conv3, "conv3") conv4 = brew.conv( model, relu3, "conv4", 512, 1024, 3, ('XavierFill', {}), ('ConstantFill', {}), pad=1 ) relu4 = brew.relu(model, conv4, "conv4") conv5 = brew.conv( model, relu4, "conv5", 1024, 1024, 3, ('XavierFill', {}), ('ConstantFill', {}), pad=1 ) relu5 = brew.relu(model, conv5, "conv5") pool5 = brew.max_pool(model, relu5, "pool5", kernel=2, stride=2) fc6 = brew.fc( model, pool5, "fc6", 1024 * 6 * 6, 3072, ('XavierFill', {}), ('ConstantFill', {}) ) relu6 = brew.relu(model, fc6, "fc6") fc7 = brew.fc( model, relu6, "fc7", 3072, 4096, ('XavierFill', {}), ('ConstantFill', {}) ) relu7 = brew.relu(model, fc7, "fc7") fc8 = brew.fc( model, relu7, "fc8", 4096, 1000, ('XavierFill', {}), ('ConstantFill', {}) ) pred = brew.softmax(model, fc8, "pred") xent = model.LabelCrossEntropy([pred, "label"], "xent") if device != 'MKL': loss = model.AveragedLoss(xent, "loss") return model, 231
def Inception(order, cudnn_ws, device): my_arg_scope = {'order': order, 'use_cudnn': True, 'cudnn_exhaustive_search': True, 'ws_nbytes_limit': str(cudnn_ws)} model = ModelHelper(name="inception", arg_scope=my_arg_scope) conv1 = brew.conv( model, "data", "conv1", 3, 64, 7, ('XavierFill', {}), ('ConstantFill', {}), stride=2, pad=3 ) relu1 = brew.relu(model, conv1, "conv1") pool1 = brew.max_pool(model, relu1, "pool1", kernel=3, stride=2, pad=1) conv2a = brew.conv( model, pool1, "conv2a", 64, 64, 1, ('XavierFill', {}), ('ConstantFill', {}) ) conv2a = brew.relu(model, conv2a, conv2a) conv2 = brew.conv( model, conv2a, "conv2", 64, 192, 3, ('XavierFill', {}), ('ConstantFill', {}), pad=1 ) relu2 = brew.relu(model, conv2, "conv2") pool2 = brew.max_pool(model, relu2, "pool2", kernel=3, stride=2, pad=1) # Inception modules inc3 = _InceptionModule( model, pool2, 192, "inc3", 64, [96, 128], [16, 32], 32 ) inc4 = _InceptionModule( model, inc3, 256, "inc4", 128, [128, 192], [32, 96], 64 ) pool5 = brew.max_pool(model, inc4, "pool5", kernel=3, stride=2, pad=1) inc5 = _InceptionModule( model, pool5, 480, "inc5", 192, [96, 208], [16, 48], 64 ) inc6 = _InceptionModule( model, inc5, 512, "inc6", 160, [112, 224], [24, 64], 64 ) inc7 = _InceptionModule( model, inc6, 512, "inc7", 128, [128, 256], [24, 64], 64 ) inc8 = _InceptionModule( model, inc7, 512, "inc8", 112, [144, 288], [32, 64], 64 ) inc9 = _InceptionModule( model, inc8, 528, "inc9", 256, [160, 320], [32, 128], 128 ) pool9 = brew.max_pool(model, inc9, "pool9", kernel=3, stride=2, pad=1) inc10 = _InceptionModule( model, pool9, 832, "inc10", 256, [160, 320], [32, 128], 128 ) inc11 = _InceptionModule( model, inc10, 832, "inc11", 384, [192, 384], [48, 128], 128 ) pool11 = brew.average_pool(model, inc11, "pool11", kernel=7, stride=1) fc = brew.fc( model, pool11, "fc", 1024, 1000, ('XavierFill', {}), ('ConstantFill', {}) ) # It seems that Soumith's benchmark does not have softmax on top # for Inception. We will add it anyway so we can have a proper # backward pass. pred = brew.softmax(model, fc, "pred") xent = model.LabelCrossEntropy([pred, "label"], "xent") if device != 'MKL': loss = model.AveragedLoss(xent, "loss") return model, 224
def VGGA(order, cudnn_ws, device): my_arg_scope = {'order': order, 'use_cudnn': True, 'cudnn_exhaustive_search': True, 'ws_nbytes_limit': str(cudnn_ws)} model = ModelHelper(name='vgg-a', arg_scope=my_arg_scope) conv1 = brew.conv( model, "data", "conv1", 3, 64, 3, ('XavierFill', {}), ('ConstantFill', {}), pad=1 ) relu1 = brew.relu(model, conv1, "conv1") pool1 = brew.max_pool(model, relu1, "pool1", kernel=2, stride=2) conv2 = brew.conv( model, pool1, "conv2", 64, 128, 3, ('XavierFill', {}), ('ConstantFill', {}), pad=1 ) relu2 = brew.relu(model, conv2, "conv2") pool2 = brew.max_pool(model, relu2, "pool2", kernel=2, stride=2) conv3 = brew.conv( model, pool2, "conv3", 128, 256, 3, ('XavierFill', {}), ('ConstantFill', {}), pad=1 ) relu3 = brew.relu(model, conv3, "conv3") conv4 = brew.conv( model, relu3, "conv4", 256, 256, 3, ('XavierFill', {}), ('ConstantFill', {}), pad=1 ) relu4 = brew.relu(model, conv4, "conv4") pool4 = brew.max_pool(model, relu4, "pool4", kernel=2, stride=2) conv5 = brew.conv( model, pool4, "conv5", 256, 512, 3, ('XavierFill', {}), ('ConstantFill', {}), pad=1 ) relu5 = brew.relu(model, conv5, "conv5") conv6 = brew.conv( model, relu5, "conv6", 512, 512, 3, ('XavierFill', {}), ('ConstantFill', {}), pad=1 ) relu6 = brew.relu(model, conv6, "conv6") pool6 = brew.max_pool(model, relu6, "pool6", kernel=2, stride=2) conv7 = brew.conv( model, pool6, "conv7", 512, 512, 3, ('XavierFill', {}), ('ConstantFill', {}), pad=1 ) relu7 = brew.relu(model, conv7, "conv7") conv8 = brew.conv( model, relu7, "conv8", 512, 512, 3, ('XavierFill', {}), ('ConstantFill', {}), pad=1 ) relu8 = brew.relu(model, conv8, "conv8") pool8 = brew.max_pool(model, relu8, "pool8", kernel=2, stride=2) fcix = brew.fc( model, pool8, "fcix", 512 * 7 * 7, 4096, ('XavierFill', {}), ('ConstantFill', {}) ) reluix = brew.relu(model, fcix, "fcix") fcx = brew.fc( model, reluix, "fcx", 4096, 4096, ('XavierFill', {}), ('ConstantFill', {}) ) relux = brew.relu(model, fcx, "fcx") fcxi = brew.fc( model, relux, "fcxi", 4096, 1000, ('XavierFill', {}), ('ConstantFill', {}) ) pred = brew.softmax(model, fcxi, "pred") xent = model.LabelCrossEntropy([pred, "label"], "xent") if device != 'MKL': loss = model.AveragedLoss(xent, "loss") return model, 231
def create_resnet50( model, data, num_input_channels, num_labels, label=None, is_test=False, no_loss=False, no_bias=0, conv1_kernel=7, conv1_stride=2, final_avg_kernel=7, ): # conv1 + maxpool brew.conv( model, data, 'conv1', num_input_channels, 64, weight_init=("MSRAFill", {}), kernel=conv1_kernel, stride=conv1_stride, pad=3, no_bias=no_bias ) brew.spatial_bn( model, 'conv1', 'conv1_spatbn_relu', 64, epsilon=1e-3, momentum=0.1, is_test=is_test ) brew.relu(model, 'conv1_spatbn_relu', 'conv1_spatbn_relu') brew.max_pool(model, 'conv1_spatbn_relu', 'pool1', kernel=3, stride=2) # Residual blocks... builder = ResNetBuilder(model, 'pool1', no_bias=no_bias, is_test=is_test, spatial_bn_mom=0.1) # conv2_x (ref Table 1 in He et al. (2015)) builder.add_bottleneck(64, 64, 256) builder.add_bottleneck(256, 64, 256) builder.add_bottleneck(256, 64, 256) # conv3_x builder.add_bottleneck(256, 128, 512, down_sampling=True) for _ in range(1, 4): builder.add_bottleneck(512, 128, 512) # conv4_x builder.add_bottleneck(512, 256, 1024, down_sampling=True) for _ in range(1, 6): builder.add_bottleneck(1024, 256, 1024) # conv5_x builder.add_bottleneck(1024, 512, 2048, down_sampling=True) builder.add_bottleneck(2048, 512, 2048) builder.add_bottleneck(2048, 512, 2048) # Final layers final_avg = brew.average_pool( model, builder.prev_blob, 'final_avg', kernel=final_avg_kernel, stride=1, ) # Final dimension of the "image" is reduced to 7x7 last_out = brew.fc( model, final_avg, 'last_out_L{}'.format(num_labels), 2048, num_labels ) if no_loss: return last_out # If we create model for training, use softmax-with-loss if (label is not None): (softmax, loss) = model.SoftmaxWithLoss( [last_out, label], ["softmax", "loss"], ) return (softmax, loss) else: # For inference, we just return softmax return brew.softmax(model, last_out, "softmax")
def alexnet(): model = ModelHelper(name="r", arg_scope={"order": "NCHW", "is_test": True}) conv1 = brew.conv( model, "data", "conv1", 3, 64, 11, ('XavierFill', {}), ('ConstantFill', {}), stride=4, pad=2 ) relu1 = brew.relu(model, conv1, "conv1") pool1 = brew.max_pool(model, relu1, "pool1", kernel=3, stride=2, pad=0, legacy_pad=3) lrn1 = brew.lrn( model, pool1, "pool1_lrn", size=5, alpha=1.0e-4, beta=0.75, bias=1.0) conv2 = brew.conv( model, lrn1, "conv2", 64, 192, 5, ('XavierFill', {}), ('ConstantFill', {}), pad=2 ) relu2 = brew.relu(model, conv2, "conv2") pool2 = brew.max_pool(model, relu2, "pool2", kernel=3, stride=2) lrn2 = brew.lrn( model, pool2, "pool2_lrn", size=5, alpha=1.0e-4, beta=0.75, bias=1.0) conv3 = brew.conv( model, lrn2, "conv3", 192, 384, 3, ('XavierFill', {}), ('ConstantFill', {}), pad=1 ) relu3 = brew.relu(model, conv3, "conv3") conv4 = brew.conv( model, relu3, "conv4", 384, 256, 3, ('XavierFill', {}), ('ConstantFill', {}), pad=1 ) relu4 = brew.relu(model, conv4, "conv4") conv5 = brew.conv( model, relu4, "conv5", 256, 256, 3, ('XavierFill', {}), ('ConstantFill', {}), pad=1 ) relu5 = brew.relu(model, conv5, "conv5") pool5 = brew.max_pool(model, relu5, "pool5", kernel=3, stride=2) fc6 = brew.fc( model, pool5, "fc6", 256 * 6 * 6, 4096, ('XavierFill', {}), ('ConstantFill', {}) ) relu6 = brew.relu(model, fc6, "fc6") fc7 = brew.fc( model, relu6, "fc7", 4096, 4096, ('XavierFill', {}), ('ConstantFill', {}) ) relu7 = brew.relu(model, fc7, "fc7") drop7 = brew.dropout(model, relu7, "fc7_dropout", is_test=1, ratio=0.5) fc8 = brew.fc( model, drop7, "fc8", 4096, 1000, ('XavierFill', {}), ('ConstantFill', {}) ) relu8 = brew.relu(model, fc8, "fc8") _ = brew.dropout(model, relu8, "fc8_dropout", is_test=1, ratio=0.5) return model, [(1, 3, 224, 224)]
def _MiniAlexNetNoDropout(self, order): # First, AlexNet using the cnn wrapper. model = model_helper.ModelHelper(name="alexnet") conv1 = brew.conv( model, "data", "conv1", 3, 16, 11, ("XavierFill", {}), ("ConstantFill", {}), stride=4, pad=0 ) relu1 = brew.relu(model, conv1, "relu1") norm1 = brew.lrn(model, relu1, "norm1", size=5, alpha=0.0001, beta=0.75) pool1 = brew.max_pool(model, norm1, "pool1", kernel=3, stride=2) conv2 = brew.group_conv( model, pool1, "conv2", 16, 32, 5, ("XavierFill", {}), ("ConstantFill", {"value": 0.1}), group=2, stride=1, pad=2 ) relu2 = brew.relu(model, conv2, "relu2") norm2 = brew.lrn(model, relu2, "norm2", size=5, alpha=0.0001, beta=0.75) pool2 = brew.max_pool(model, norm2, "pool2", kernel=3, stride=2) conv3 = brew.conv( model, pool2, "conv3", 32, 64, 3, ("XavierFill", {'std': 0.01}), ("ConstantFill", {}), pad=1 ) relu3 = brew.relu(model, conv3, "relu3") conv4 = brew.group_conv( model, relu3, "conv4", 64, 64, 3, ("XavierFill", {}), ("ConstantFill", {"value": 0.1}), group=2, pad=1 ) relu4 = brew.relu(model, conv4, "relu4") conv5 = brew.group_conv( model, relu4, "conv5", 64, 32, 3, ("XavierFill", {}), ("ConstantFill", {"value": 0.1}), group=2, pad=1 ) relu5 = brew.relu(model, conv5, "relu5") pool5 = brew.max_pool(model, relu5, "pool5", kernel=3, stride=2) fc6 = brew.fc( model, pool5, "fc6", 1152, 1024, ("XavierFill", {}), ("ConstantFill", {"value": 0.1}) ) relu6 = brew.relu(model, fc6, "relu6") fc7 = brew.fc( model, relu6, "fc7", 1024, 1024, ("XavierFill", {}), ("ConstantFill", {"value": 0.1}) ) relu7 = brew.relu(model, fc7, "relu7") fc8 = brew.fc( model, relu7, "fc8", 1024, 5, ("XavierFill", {}), ("ConstantFill", {"value": 0.0}) ) pred = brew.softmax(model, fc8, "pred") xent = model.LabelCrossEntropy([pred, "label"], "xent") loss = model.AveragedLoss([xent], ["loss"]) model.AddGradientOperators([loss]) return model