def test_arg_scope(self):
myhelper = self.myhelper
myhelper2 = self.myhelper2
n = 15
with brew.arg_scope([myhelper], val=n):
res = brew.myhelper(self.model)
self.assertEqual(n, res)
with brew.arg_scope([myhelper, myhelper2], val=n):
res1 = brew.myhelper(self.model)
res2 = brew.myhelper2(self.model)
self.assertEqual([n, n], [res1, res2])
def test_arg_scope(self):
myhelper = self.myhelper
myhelper2 = self.myhelper2
n = 15
with brew.arg_scope([myhelper], val=n):
res = brew.myhelper(self.model)
self.assertEqual(n, res)
with brew.arg_scope([myhelper, myhelper2], val=n):
res1 = brew.myhelper(self.model)
res2 = brew.myhelper2(self.model)
self.assertEqual([n, n], [res1, res2])
def test_arg_scope_nested(self):
myhelper = self.myhelper
n = 16
with brew.arg_scope([myhelper], val=-3), \
brew.arg_scope([myhelper], val=-2):
with brew.arg_scope([myhelper], val=n):
res = brew.myhelper(None)
self.assertEqual(n, res)
res = brew.myhelper(None)
self.assertEqual(res, -2)
res = brew.myhelper(None, val=15)
self.assertEqual(res, 15)
def test_arg_scope_nested(self):
myhelper = self.myhelper
n = 16
with brew.arg_scope([myhelper], val=-3), \
brew.arg_scope([myhelper], val=-2):
with brew.arg_scope([myhelper], val=n):
res = brew.myhelper(self.model)
self.assertEqual(n, res)
res = brew.myhelper(self.model)
self.assertEqual(res, -2)
res = brew.myhelper(self.model, val=15)
self.model.Validate()
self.assertEqual(res, 15)
def create_model(model, loss_scale):
initializer = (PseudoFP16Initializer
if args.data_type == 'float16' else Initializer)
with brew.arg_scope([brew.conv, brew.fc],
WeightInitializer=initializer,
BiasInitializer=initializer,
enable_tensor_core=False,
float16_compute=False):
pred = resnet.create_resnet50(
model,
"data",
num_input_channels=args.channels,
num_labels=args.num_labels,
# num_groups=args.resnext_num_groups,
# num_width_per_group=args.resnext_width_per_group,
no_bias=True,
no_loss=True)
# If we're using float on 2B, then inflate to the 4B representation
if args.data_type == 'float16':
pred = model.net.HalfToFloat(pred, pred + '_fp32')
# Compute the softmax probabilities and the loss
softmax, loss = model.SoftmaxWithLoss([pred, 'label'],
['softmax', 'loss'])
# Noralize the loss, and compute the top_k accuracies for k \in {1, 5}
loss = model.Scale(loss, scale=loss_scale)
brew.accuracy(model, [softmax, "label"], "accuracy", top_k=1)
brew.accuracy(model, [softmax, "label"], "accuracy_top5", top_k=5)
return [loss]
def create_sphereface_model_ops(model, loss_scale):
initializer = (pFP16Initializer if args.dtype == 'float16'
else Initializer)
with brew.arg_scope([brew.conv, brew.fc],
WeightInitializer=initializer,
BiasInitializer=initializer,
enable_tensor_core=args.enable_tensor_core):
pred = sphereface.create_net(
model,
"data",
"label",
in_dim=args.num_channels,
class_num=args.num_labels,
feature_dim=args.feature_dim,
is_test=False,
no_loss=True,
fp16_data=True if args.dtype == 'float16' else False,
)
if args.dtype == 'float16':
pred = model.net.HalfToFloat(pred, pred + '_fp32')
softmax, loss = model.SoftmaxWithLoss([pred, 'label'],
['softmax', 'loss'])
loss = model.Scale(loss, scale=loss_scale)
brew.accuracy(model, [softmax, "label"], "accuracy")
return [loss]
def create_resnext_model_ops(model, loss_scale):
initializer = (PseudoFP16Initializer if args.dtype == 'float16'
else Initializer)
with brew.arg_scope([brew.conv, brew.fc],
WeightInitializer=initializer,
BiasInitializer=initializer,
enable_tensor_core=args.enable_tensor_core,
float16_compute=args.float16_compute):
pred = resnet.create_resnext(
model,
"data",
num_input_channels=args.num_channels,
num_labels=args.num_labels,
num_layers=args.num_layers,
num_groups=args.resnext_num_groups,
num_width_per_group=args.resnext_width_per_group,
no_bias=True,
no_loss=True,
)
if args.dtype == 'float16':
pred = model.net.HalfToFloat(pred, pred + '_fp32')
softmax, loss = model.SoftmaxWithLoss([pred, 'label'],
['softmax', 'loss'])
loss = model.Scale(loss, scale=loss_scale)
brew.accuracy(model, [softmax, "label"], "accuracy", top_k=1)
brew.accuracy(model, [softmax, "label"], "accuracy_top5", top_k=5)
return [loss]
def test_model_helper(self):
X = np.random.rand(64, 32, 32, 3).astype(np.float32) - 0.5
workspace.FeedBlob("x", X)
my_arg_scope = {'order': 'NHWC'}
model = ModelHelper(name="test_model", arg_scope=my_arg_scope)
with brew.arg_scope(
brew.conv,
stride=2,
pad=2,
weight_init=('XavierFill', {}),
bias_init=('ConstantFill', {})
):
brew.conv(
model=model,
blob_in="x",
blob_out="out",
dim_in=3,
dim_out=64,
kernel=3,
)
model.Validate()
workspace.RunNetOnce(model.param_init_net)
workspace.RunNetOnce(model.net)
out = workspace.FetchBlob("out")
self.assertEqual(out.shape, (64, 17, 17, 64))
def create_resnet50_model_ops(model, loss_scale):
initializer = (pFP16Initializer if args.dtype == 'float16'
else Initializer)
with brew.arg_scope([brew.conv, brew.fc],
WeightInitializer=initializer,
BiasInitializer=initializer,
enable_tensor_core=args.enable_tensor_core):
pred = resnet.create_resnet50(
model,
"data",
num_input_channels=args.num_channels,
num_labels=args.num_labels,
no_bias=True,
no_loss=True,
)
if args.dtype == 'float16':
pred = model.net.HalfToFloat(pred, pred + '_fp32')
softmax, loss = model.SoftmaxWithLoss([pred, 'label'],
['softmax', 'loss'])
loss = model.Scale(loss, scale=loss_scale)
brew.accuracy(model, [softmax, "label"], "accuracy")
return [loss]
def Resnext101(model, loss_scale, dtype='float'):
initializer = (PseudoFP16Initializer
if dtype == 'float16' else Initializer)
with brew.arg_scope(
[brew.conv, brew.fc],
WeightInitializer=initializer,
BiasInitializer=initializer,
):
# residual network
pred = resnet.create_resnext(
model,
"data",
num_input_channels=3,
num_labels=1000,
label="label",
num_layers=101,
num_groups=32,
num_width_per_group=4,
no_bias=True,
no_loss=True,
)
if dtype == 'float16':
pred = model.net.HalfToFloat(pred, pred + '_fp32')
softmax, loss = model.SoftmaxWithLoss([pred, 'label'],
['softmax', 'loss'])
prefix = model.net.Proto().name
loss = model.net.Scale(loss, prefix + "_loss", scale=loss_scale)
brew.accuracy(model, [softmax, "label"], prefix + "_accuracy")
return [loss]
def test_arg_scope_single(self):
X = np.random.rand(64, 3, 32, 32).astype(np.float32) - 0.5
workspace.FeedBlob("x", X)
model = ModelHelper(name="test_model")
with brew.arg_scope(
brew.conv,
stride=2,
pad=2,
weight_init=('XavierFill', {}),
bias_init=('ConstantFill', {})
):
brew.conv(
model=model,
blob_in="x",
blob_out="out",
dim_in=3,
dim_out=64,
kernel=3,
)
model.Validate()
workspace.RunNetOnce(model.param_init_net)
workspace.RunNetOnce(model.net)
out = workspace.FetchBlob("out")
self.assertEqual(out.shape, (64, 64, 17, 17))
def resnet152_model(self, model, data, precision):
initializer = Initializer if precision == 'fp32' else pFP16Initializer
with brew.arg_scope([brew.conv, brew.fc],
WeightInitializer=initializer,
BiasInitializer=initializer,
enable_tensor_core=True):
model = create_resnet152(model, "data", 3, 1000)
return model
def vgg16_model(self, model, data, precision):
initializer = Initializer if precision == 'fp32' else PseudoFP16Initializer
with brew.arg_scope([brew.conv, brew.fc],
WeightInitializer=initializer,
BiasInitializer=initializer,
enable_tensor_core=True):
conv1_1 = brew.conv(model, data, 'conv1_1', dim_in=3, dim_out=64, kernel=3, pad=1)
conv1_1 = brew.relu(model, conv1_1, conv1_1)
conv1_2 = brew.conv(model, conv1_1, 'conv1_2', dim_in=64, dim_out=64, kernel=3, pad=1)
conv1_2 = brew.relu(model, conv1_2, conv1_2)
pool1 = brew.max_pool(model, conv1_2, 'pool1', kernel=2, stride=2)
conv2_1 = brew.conv(model, pool1, 'conv2_1', dim_in=64, dim_out=128, kernel=3, pad=1)
conv2_1 = brew.relu(model, conv2_1, conv2_1)
conv2_2 = brew.conv(model, conv2_1, 'conv2_2', dim_in=128, dim_out=128, kernel=3, pad=1)
conv2_2 = brew.relu(model, conv2_2, conv2_2)
pool2 = brew.max_pool(model, conv2_2, 'pool2', kernel=2, stride=2)
conv3_1 = brew.conv(model, pool2, 'conv3_1', dim_in=128, dim_out=256, kernel=3, pad=1)
conv3_1 = brew.relu(model, conv3_1, conv3_1)
conv3_2 = brew.conv(model, conv3_1, 'conv3_2', dim_in=256, dim_out=256, kernel=3, pad=1)
conv3_2 = brew.relu(model, conv3_2, conv3_2)
conv3_3 = brew.conv(model, conv3_2, 'conv3_3', dim_in=256, dim_out=256, kernel=3, pad=1)
conv3_3 = brew.relu(model, conv3_3, conv3_3)
pool3 = brew.max_pool(model, conv3_3, 'pool3', kernel=2, stride=2)
conv4_1 = brew.conv(model, pool3, 'conv4_1', dim_in=256, dim_out=512, kernel=3, pad=1)
conv4_1 = brew.relu(model, conv4_1, conv4_1)
conv4_2 = brew.conv(model, conv4_1, 'conv4_2', dim_in=512, dim_out=512, kernel=3, pad=1)
conv4_2 = brew.relu(model, conv4_2, conv4_2)
conv4_3 = brew.conv(model, conv4_2, 'conv4_3', dim_in=512, dim_out=512, kernel=3, pad=1)
conv4_3 = brew.relu(model, conv4_3, conv4_3)
pool4 = brew.max_pool(model, conv4_3, 'pool4', kernel=2, stride=2)
conv5_1 = brew.conv(model, pool4, 'conv5_1', dim_in=512, dim_out=512, kernel=3, pad=1)
conv5_1 = brew.relu(model, conv5_1, conv5_1)
conv5_2 = brew.conv(model, conv5_1, 'conv5_2', dim_in=512, dim_out=512, kernel=3, pad=1)
conv5_2 = brew.relu(model, conv5_2, conv5_2)
conv5_3 = brew.conv(model, conv5_2, 'conv5_3', dim_in=512, dim_out=512, kernel=3, pad=1)
conv5_3 = brew.relu(model, conv5_3, conv5_3)
pool5 = brew.max_pool(model, conv5_3, 'pool5', kernel=2, stride=2)
fc6 = brew.fc(model, pool5, 'fc6', dim_in=25088, dim_out=4096)
fc6 = brew.relu(model, fc6, fc6)
fc7 = brew.fc(model, fc6, 'fc7', dim_in=4096, dim_out=4096)
fc7 = brew.relu(model, fc7, fc7)
pred = brew.fc(model, fc7, 'pred', 4096, 1000)
softmax = brew.softmax(model, pred, 'softmax')
return softmax
def create_target_model_ops(model, loss_scale):
initializer = (PseudoFP16Initializer if args.dtype == 'float16'
else Initializer)
with brew.arg_scope([brew.conv, brew.fc],
WeightInitializer=initializer,
BiasInitializer=initializer,
enable_tensor_core=args.enable_tensor_core,
float16_compute=args.float16_compute):
pred = add_se_model(model, model_config, "data", is_test=False)
if args.dtype == 'float16':
pred = model.net.HalfToFloat(pred, pred + '_fp32')
loss = add_softmax_loss(model, pred, 'label')
brew.accuracy(model, ['softmax', 'label'], 'accuracy')
return [loss]
def AddForwardPassOps(model):
"""Add forward pass ops and return a list of losses."""
with brew.arg_scope([brew.conv, brew.fc],
WeightInitializer=pFP16Initializer,
BiasInitializer=pFP16Initializer):
conv1 = brew.conv(model, 'data', 'conv1', 1, 20, 5)
pool1 = brew.max_pool(model, conv1, 'pool1', kernel=2, stride=2)
conv2 = brew.conv(model, pool1, 'conv2', 20, 50, 5)
pool2 = brew.max_pool(model, conv2, 'pool2', kernel=2, stride=2)
fc3 = brew.fc(model, pool2, 'fc3', 50 * 4 * 4, 500)
fc3 = brew.relu(model, fc3, fc3)
pred = brew.fc(model, fc3, 'pred', 500, 10)
# Cast back to fp32 for remaining ops
pred = model.net.HalfToFloat(pred, pred + '_fp32')
softmax, loss = model.SoftmaxWithLoss([pred, 'label'], ['softmax', 'loss'])
brew.accuracy(model, [softmax, 'label'], 'accuracy')
return [loss]
def create_model(model_builder, model, enable_tensor_core, float16_compute, loss_scale=1.0):
"""Creates one model replica.
:param obj model_builder: A model instance that contains `forward_pass_builder` method.
:param model: Caffe2's model helper class instances.
:type model: :py:class:`caffe2.python.model_helper.ModelHelper`
:param bool enable_tensor_core: If true, Volta's tensor core ops are enabled.
:param float loss_scale: Scale loss for multi-GPU training.
:return: Head nodes (softmax or loss depending on phase)
"""
initializer = (pFP16Initializer if model_builder.dtype == 'float16' else Initializer)
with brew.arg_scope([brew.conv, brew.fc],
WeightInitializer=initializer,
BiasInitializer=initializer,
enable_tensor_core=enable_tensor_core,
float16_compute=float16_compute):
outputs = model_builder.forward_pass_builder(model, loss_scale=loss_scale)
return outputs
def create_resnet50_model_ops(model, loss_scale):
with brew.arg_scope([brew.conv, brew.fc],
WeightInitializer=Initializer,
BiasInitializer=Initializer,
enable_tensor_core=0):
pred = resnet.create_resnet50(
model,
"data",
num_input_channels=num_channels,
num_labels=num_labels,
no_bias=True,
no_loss=True,
)
softmax, loss = model.SoftmaxWithLoss([pred, 'label'],
['softmax', 'loss'])
loss = model.Scale(loss, scale=loss_scale)
brew.accuracy(model, [softmax, "label"], "accuracy")
return [loss]
def create_model_ops_testable(model, loss_scale, is_test=False):
initializer = (PseudoFP16Initializer
if args.dtype == 'float16' else Initializer)
with brew.arg_scope([brew.conv, brew.fc],
WeightInitializer=initializer,
BiasInitializer=initializer,
enable_tensor_core=args.enable_tensor_core,
float16_compute=args.float16_compute):
if args.model == "cifar10":
if args.image_size != 32:
log.warn("Cifar10 expects a 32x32 image.")
pred = models.cifar10.create_cifar10(
model,
"data",
image_channels=args.num_channels,
num_classes=args.num_labels,
image_height=args.image_size,
image_width=args.image_size,
)
elif args.model == "resnet32x32":
if args.image_size != 32:
log.warn("ResNet32x32 expects a 32x32 image.")
pred = models.resnet.create_resnet32x32(
model,
"data",
num_layers=args.num_layers,
num_input_channels=args.num_channels,
num_labels=args.num_labels,
is_test=is_test)
elif args.model == "resnet":
if args.image_size != 224:
log.warn(
"ResNet expects a 224x224 image. input image = %d" %
args.image_size)
pred = resnet.create_resnet50(
#args.layers,
model,
"data",
num_input_channels=args.num_channels,
num_labels=args.num_labels,
no_bias=True,
no_loss=True,
)
elif args.model == "vgg":
if args.image_size != 224:
log.warn("VGG expects a 224x224 image.")
pred = vgg.create_vgg(model,
"data",
num_input_channels=args.num_channels,
num_labels=args.num_labels,
num_layers=args.num_layers,
is_test=is_test)
elif args.model == "googlenet":
if args.image_size != 224:
log.warn("GoogLeNet expects a 224x224 image.")
pred = googlenet.create_googlenet(
model,
"data",
num_input_channels=args.num_channels,
num_labels=args.num_labels,
is_test=is_test)
elif args.model == "alexnet":
if args.image_size != 224:
log.warn("Alexnet expects a 224x224 image.")
pred = alexnet.create_alexnet(
model,
"data",
num_input_channels=args.num_channels,
num_labels=args.num_labels,
is_test=is_test)
elif args.model == "alexnetv0":
if args.image_size != 224:
log.warn("Alexnet v0 expects a 224x224 image.")
pred = alexnet.create_alexnetv0(
model,
"data",
num_input_channels=args.num_channels,
num_labels=args.num_labels,
is_test=is_test)
else:
raise NotImplementedError("Network {} not found.".format(
args.model))
if args.dtype == 'float16':
pred = model.net.HalfToFloat(pred, pred + '_fp32')
softmax, loss = model.SoftmaxWithLoss([pred, 'label'],
['softmax', 'loss'])
loss = model.Scale(loss, scale=loss_scale)
brew.accuracy(model, [softmax, "label"], "accuracy")
return [loss]
def AddForwardPassOps(model, loss_scale, dtype):
"""Add forward pass ops and return a list of losses."""
initializer = (pFP16Initializer
if dtype == DataType.FLOAT16 else Initializer)
with brew.arg_scope([brew.conv, brew.fc],
WeightInitializer=initializer,
BiasInitializer=initializer):
conv1 = brew.conv(model,
'data',
'conv1',
3,
32,
5,
pad=2,
weight_init=('GaussianFill', {
'std': 0.0001,
'mean': 0.0
}))
pool1 = brew.max_pool(model, conv1, 'pool1', kernel=3, stride=2)
relu1 = brew.relu(model, pool1, 'relu1')
conv2 = brew.conv(model,
relu1,
'conv2',
32,
32,
5,
pad=2,
weight_init=('GaussianFill', {
'std': 0.01
}))
conv2 = brew.relu(model, conv2, conv2)
pool2 = brew.average_pool(model, conv2, 'pool2', kernel=3, stride=2)
conv3 = brew.conv(model,
pool2,
'conv3',
32,
64,
5,
pad=2,
weight_init=('GaussianFill', {
'std': 0.01
}))
conv3 = brew.relu(model, conv3, conv3)
pool3 = brew.average_pool(model, conv3, 'pool3', kernel=3, stride=2)
fc1 = brew.fc(model,
pool3,
'fc1',
64 * 3 * 3,
64,
weight_init=('GaussianFill', {
'std': 0.1
}))
fc2 = brew.fc(model,
fc1,
'fc2',
64,
10,
weight_init=('GaussianFill', {
'std': 0.1
}))
if dtype == DataType.FLOAT16:
fc2 = model.net.HalfToFloat(fc2, fc2 + '_fp32')
softmax, loss = model.SoftmaxWithLoss([fc2, 'label'], ['softmax', 'loss'])
loss = model.Scale(loss, loss, scale=loss_scale)
brew.accuracy(model, [softmax, 'label'], 'accuracy')
return [loss]
def CivilNet(name, train_test_deplopy=0):
arg_scope = {
'order': 'NCHW',
'use_cudnn': True,
'cudnn_exhaustive_search': True,
'ws_nbytes_limit': (64 * 1024 * 1024)
}
model = model_helper.ModelHelper(name=name, arg_scope=arg_scope)
model._device_type = caffe2_pb2.CUDA
model._device_prefix = "gpu"
model._shared_model = False
model._devices = [0]
device_opt = core.DeviceOption(caffe2_pb2.CUDA, 0)
#for deploy
if train_test_deplopy == 2:
with core.DeviceScope(device_opt):
with core.NameScope("{}_{}".format(model._device_prefix, 0)):
with brew.arg_scope([brew.conv, brew.fc],
WeightInitializer=Initializer,
BiasInitializer=Initializer,
enable_tensor_core=False,
float16_compute=False):
resnet.create_resnet50(model,
"data",
num_input_channels=3,
num_labels=args.num_labels,
no_bias=True,
no_loss=False)
workspace.RunNetOnce(model.param_init_net)
workspace.CreateNet(model.net)
return model
reader_name = "reader" if train_test_deplopy == 0 else "test_reader"
reader_data = args.train_data if train_test_deplopy == 0 else args.test_data
reader = model.CreateDB(reader_name,
db=reader_data,
db_type='lmdb',
num_shards=1,
shard_id=0)
is_test = True if train_test_deplopy == 1 else False
loss = None
with core.DeviceScope(device_opt):
with core.NameScope("{}_{}".format(model._device_prefix, 0)):
AddImageInput(model, reader, batch_size=32, is_test=is_test)
with brew.arg_scope([brew.conv, brew.fc],
WeightInitializer=Initializer,
BiasInitializer=Initializer,
enable_tensor_core=False,
float16_compute=False):
pred = resnet.create_resnet50(model,
"data",
num_input_channels=3,
num_labels=args.num_labels,
no_bias=True,
no_loss=True)
softmax, loss = model.SoftmaxWithLoss([pred, 'label'],
['softmax', 'loss'])
brew.accuracy(model, [softmax, "label"], "accuracy")
#for test
if train_test_deplopy == 1:
workspace.RunNetOnce(model.param_init_net)
workspace.CreateNet(model.net)
return model
#for train
loss_grad = {}
losses_by_gpu = {}
losses_by_gpu[0] = [loss]
#add grad
def create_grad(lossp):
return model.ConstantFill(lossp, str(lossp) + "_grad", value=1.0)
# Explicitly need to create gradients on GPU 0
device = core.DeviceOption(model._device_type, 0)
with core.DeviceScope(device):
for l in losses_by_gpu[0]:
lg = create_grad(l)
loss_grad[str(l)] = str(lg)
model.AddGradientOperators(loss_grad)
#end add grad
optimizer.add_weight_decay(model, args.weight_decay)
stepsz = int(30 * args.epoch_size / 32)
opt = optimizer.build_multi_precision_sgd(model,
args.base_learning_rate,
momentum=0.9,
nesterov=1,
policy="step",
stepsize=stepsz,
gamma=0.1)
model._optimizer = opt
workspace.RunNetOnce(model.param_init_net)
workspace.CreateNet(model.net)
return model
def Inception(model, loss_scale, dtype='float'):
initializer = (PseudoFP16Initializer
if dtype == 'float16' else Initializer)
with brew.arg_scope(
[brew.conv, brew.fc],
WeightInitializer=initializer,
BiasInitializer=initializer,
):
conv1 = brew.conv(
model,
"data",
"conv1",
3,
64,
7,
('XavierFill', {}),
('ConstantFill', {}),
stride=2,
pad=3,
)
relu1 = brew.relu(model, conv1, "conv1")
pool1 = brew.max_pool(model, relu1, "pool1", kernel=3, stride=2, pad=1)
conv2a = brew.conv(model, pool1, "conv2a", 64, 64, 1,
('XavierFill', {}), ('ConstantFill', {}))
conv2a = brew.relu(model, conv2a, conv2a)
conv2 = brew.conv(
model,
conv2a,
"conv2",
64,
192,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1,
)
relu2 = brew.relu(model, conv2, "conv2")
pool2 = brew.max_pool(model, relu2, "pool2", kernel=3, stride=2, pad=1)
# Inception modules
inc3 = _InceptionModule(model, pool2, 192, "inc3", 64, [96, 128],
[16, 32], 32)
inc4 = _InceptionModule(model, inc3, 256, "inc4", 128, [128, 192],
[32, 96], 64)
pool5 = brew.max_pool(model, inc4, "pool5", kernel=3, stride=2, pad=1)
inc5 = _InceptionModule(model, pool5, 480, "inc5", 192, [96, 208],
[16, 48], 64)
inc6 = _InceptionModule(model, inc5, 512, "inc6", 160, [112, 224],
[24, 64], 64)
inc7 = _InceptionModule(model, inc6, 512, "inc7", 128, [128, 256],
[24, 64], 64)
inc8 = _InceptionModule(model, inc7, 512, "inc8", 112, [144, 288],
[32, 64], 64)
inc9 = _InceptionModule(model, inc8, 528, "inc9", 256, [160, 320],
[32, 128], 128)
pool9 = brew.max_pool(model, inc9, "pool9", kernel=3, stride=2, pad=1)
inc10 = _InceptionModule(model, pool9, 832, "inc10", 256, [160, 320],
[32, 128], 128)
inc11 = _InceptionModule(model, inc10, 832, "inc11", 384, [192, 384],
[48, 128], 128)
pool11 = brew.average_pool(model, inc11, "pool11", kernel=7, stride=1)
fc = brew.fc(model, pool11, "fc", 1024, 1000, ('XavierFill', {}),
('ConstantFill', {}))
if dtype == 'float16':
fc = model.net.HalfToFloat(fc, fc + '_fp32')
# It seems that Soumith's benchmark does not have softmax on top
# for Inception. We will add it anyway so we can have a proper
# backward pass.
pred = brew.softmax(model, fc, "pred")
xent = model.net.LabelCrossEntropy([pred, "label"], "xent")
loss = model.net.AveragedLoss(xent, "loss")
return [loss]
def VGGA(model, loss_scale, dtype='float'):
initializer = (PseudoFP16Initializer
if dtype == 'float16' else Initializer)
with brew.arg_scope(
[brew.conv, brew.fc],
WeightInitializer=initializer,
BiasInitializer=initializer,
):
conv1 = brew.conv(
model,
"data",
"conv1",
3,
64,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1,
)
relu1 = brew.relu(model, conv1, "conv1")
pool1 = brew.max_pool(model, relu1, "pool1", kernel=2, stride=2)
conv2 = brew.conv(
model,
pool1,
"conv2",
64,
128,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1,
)
relu2 = brew.relu(model, conv2, "conv2")
pool2 = brew.max_pool(model, relu2, "pool2", kernel=2, stride=2)
conv3 = brew.conv(
model,
pool2,
"conv3",
128,
256,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1,
)
relu3 = brew.relu(model, conv3, "conv3")
conv4 = brew.conv(
model,
relu3,
"conv4",
256,
256,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1,
)
relu4 = brew.relu(model, conv4, "conv4")
pool4 = brew.max_pool(model, relu4, "pool4", kernel=2, stride=2)
conv5 = brew.conv(
model,
pool4,
"conv5",
256,
512,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1,
)
relu5 = brew.relu(model, conv5, "conv5")
conv6 = brew.conv(
model,
relu5,
"conv6",
512,
512,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1,
)
relu6 = brew.relu(model, conv6, "conv6")
pool6 = brew.max_pool(model, relu6, "pool6", kernel=2, stride=2)
conv7 = brew.conv(
model,
pool6,
"conv7",
512,
512,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1,
)
relu7 = brew.relu(model, conv7, "conv7")
conv8 = brew.conv(
model,
relu7,
"conv8",
512,
512,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1,
)
relu8 = brew.relu(model, conv8, "conv8")
pool8 = brew.max_pool(model, relu8, "pool8", kernel=2, stride=2)
fcix = brew.fc(model, pool8, "fcix", 512 * 7 * 7, 4096,
('XavierFill', {}), ('ConstantFill', {}))
reluix = brew.relu(model, fcix, "fcix")
fcx = brew.fc(model, reluix, "fcx", 4096, 4096, ('XavierFill', {}),
('ConstantFill', {}))
relux = brew.relu(model, fcx, "fcx")
fcxi = brew.fc(model, relux, "fcxi", 4096, 1000, ('XavierFill', {}),
('ConstantFill', {}))
if dtype == 'float16':
fcxi = model.net.HalfToFloat(fcxi, fcxi + '_fp32')
pred = brew.softmax(model, fcxi, "pred")
xent = model.net.LabelCrossEntropy([pred, "label"], "xent")
loss = model.net.AveragedLoss(xent, "loss")
return [loss]
def OverFeat(model, loss_scale, dtype='float'):
initializer = (PseudoFP16Initializer
if dtype == 'float16' else Initializer)
with brew.arg_scope(
[brew.conv, brew.fc],
WeightInitializer=initializer,
BiasInitializer=initializer,
):
conv1 = brew.conv(
model,
"data",
"conv1",
3,
96,
11,
('XavierFill', {}),
('ConstantFill', {}),
stride=4,
)
relu1 = brew.relu(model, conv1, "conv1")
pool1 = brew.max_pool(model, relu1, "pool1", kernel=2, stride=2)
conv2 = brew.conv(model, pool1, "conv2", 96, 256, 5,
('XavierFill', {}), ('ConstantFill', {}))
relu2 = brew.relu(model, conv2, "conv2")
pool2 = brew.max_pool(model, relu2, "pool2", kernel=2, stride=2)
conv3 = brew.conv(
model,
pool2,
"conv3",
256,
512,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1,
)
relu3 = brew.relu(model, conv3, "conv3")
conv4 = brew.conv(
model,
relu3,
"conv4",
512,
1024,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1,
)
relu4 = brew.relu(model, conv4, "conv4")
conv5 = brew.conv(
model,
relu4,
"conv5",
1024,
1024,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1,
)
relu5 = brew.relu(model, conv5, "conv5")
pool5 = brew.max_pool(model, relu5, "pool5", kernel=2, stride=2)
fc6 = brew.fc(model, pool5, "fc6", 1024 * 6 * 6, 3072,
('XavierFill', {}), ('ConstantFill', {}))
relu6 = brew.relu(model, fc6, "fc6")
fc7 = brew.fc(model, relu6, "fc7", 3072, 4096, ('XavierFill', {}),
('ConstantFill', {}))
relu7 = brew.relu(model, fc7, "fc7")
fc8 = brew.fc(model, relu7, "fc8", 4096, 1000, ('XavierFill', {}),
('ConstantFill', {}))
if dtype == 'float16':
fc8 = model.net.HalfToFloat(fc8, fc8 + '_fp32')
pred = brew.softmax(model, fc8, "pred")
xent = model.net.LabelCrossEntropy([pred, "label"], "xent")
loss = model.net.AveragedLoss(xent, "loss")
return [loss]
