def test_mobile_exporter_datatypes(self):
model = ModelHelper(name="mobile_exporter_test_model")
model.Copy("data_int", "out")
model.params.append("data_int")
model.Copy("data_obj", "out_obj")
model.params.append("data_obj")
# Create our mobile exportable networks
workspace.RunNetOnce(model.param_init_net)
np_data_int = np.random.randint(100,
size=(1, 1, 28, 28),
dtype=np.int32)
workspace.FeedBlob("data_int", np_data_int)
np_data_obj = np.array(['aa', 'bb']).astype(np.dtype('O'))
workspace.FeedBlob("data_obj", np_data_obj)
init_net, predict_net = mobile_exporter.Export(workspace, model.net,
model.params)
workspace.CreateNet(model.net)
workspace.RunNet(model.net)
ref_out = workspace.FetchBlob("out")
ref_out_obj = workspace.FetchBlob("out_obj")
# Clear the workspace
workspace.ResetWorkspace()
# Populate the workspace with data
workspace.RunNetOnce(init_net)
# Overwrite the old net
workspace.CreateNet(predict_net, True)
workspace.RunNet(predict_net.name)
manual_run_out = workspace.FetchBlob("out")
manual_run_out_obj = workspace.FetchBlob("out_obj")
np.testing.assert_allclose(ref_out,
manual_run_out,
atol=1e-10,
rtol=1e-10)
np.testing.assert_equal(ref_out_obj, manual_run_out_obj)
# Clear the workspace
workspace.ResetWorkspace()
# Predictor interface test (simulates writing to disk)
predictor = workspace.Predictor(init_net.SerializeToString(),
predict_net.SerializeToString())
# Output is a vector of outputs.
predictor_out = predictor.run([])
assert len(predictor_out) == 2
predictor_out_int = predictor_out[1]
predictor_out_obj = predictor_out[0]
# The order in predictor_out is non-deterministic. Use type of the entry
# to figure out what to compare it to.
if isinstance(predictor_out[1][0], bytes):
predictor_out_int = predictor_out[0]
predictor_out_obj = predictor_out[1]
np.testing.assert_allclose(ref_out,
predictor_out_int,
atol=1e-10,
rtol=1e-10)
np.testing.assert_equal(ref_out_obj, predictor_out_obj)
def lstm_with_attention(
self,
create_lstm_with_attention,
encoder_output_length,
encoder_output_dim,
decoder_input_length,
decoder_state_dim,
batch_size,
ref,
gc,
):
model = ModelHelper(name='external')
with core.DeviceScope(gc):
(
encoder_outputs,
decoder_inputs,
decoder_input_lengths,
initial_decoder_hidden_state,
initial_decoder_cell_state,
initial_attention_weighted_encoder_context,
) = model.net.AddExternalInputs(
'encoder_outputs',
'decoder_inputs',
'decoder_input_lengths',
'initial_decoder_hidden_state',
'initial_decoder_cell_state',
'initial_attention_weighted_encoder_context',
)
create_lstm_with_attention(
model=model,
decoder_inputs=decoder_inputs,
decoder_input_lengths=decoder_input_lengths,
initial_decoder_hidden_state=initial_decoder_hidden_state,
initial_decoder_cell_state=initial_decoder_cell_state,
initial_attention_weighted_encoder_context=(
initial_attention_weighted_encoder_context),
encoder_output_dim=encoder_output_dim,
encoder_outputs=encoder_outputs,
decoder_input_dim=decoder_state_dim,
decoder_state_dim=decoder_state_dim,
scope='external/LSTMWithAttention',
)
op = model.net._net.op[-2]
workspace.RunNetOnce(model.param_init_net)
# This is original decoder_inputs after linear layer
decoder_input_blob = op.input[0]
workspace.FeedBlob(
decoder_input_blob,
np.random.randn(
decoder_input_length,
batch_size,
decoder_state_dim * 4,
).astype(np.float32))
workspace.FeedBlob(
'external/LSTMWithAttention/encoder_outputs_transposed',
np.random.randn(
batch_size,
encoder_output_dim,
encoder_output_length,
).astype(np.float32),
)
workspace.FeedBlob(
'external/LSTMWithAttention/weighted_encoder_outputs',
np.random.randn(
encoder_output_length,
batch_size,
encoder_output_dim,
).astype(np.float32),
)
workspace.FeedBlob(
decoder_input_lengths,
np.random.randint(0, decoder_input_length + 1,
size=(batch_size, )).astype(np.int32))
workspace.FeedBlob(
initial_decoder_hidden_state,
np.random.randn(1, batch_size,
decoder_state_dim).astype(np.float32))
workspace.FeedBlob(
initial_decoder_cell_state,
np.random.randn(1, batch_size,
decoder_state_dim).astype(np.float32))
workspace.FeedBlob(
initial_attention_weighted_encoder_context,
np.random.randn(1, batch_size,
encoder_output_dim).astype(np.float32))
inputs = [workspace.FetchBlob(name) for name in op.input]
self.assertReferenceChecks(
device_option=gc,
op=op,
inputs=inputs,
reference=ref,
grad_reference=None,
output_to_grad=None,
outputs_to_check=list(range(6)),
)
gradients_to_check = [
index for (index, input_name) in enumerate(op.input)
if input_name != 'decoder_input_lengths'
]
for param in gradients_to_check:
self.assertGradientChecks(
device_option=gc,
op=op,
inputs=inputs,
outputs_to_check=param,
outputs_with_grads=[0, 4],
threshold=0.01,
stepsize=0.001,
)
def AlexNet(order, cudnn_ws, ideep):
my_arg_scope = {
'order': order,
'use_cudnn': True,
'cudnn_exhaustive_search': True,
'ws_nbytes_limit': str(cudnn_ws)
}
model = ModelHelper(name="alexnet", arg_scope=my_arg_scope)
conv1 = brew.conv(model,
"data",
"conv1",
3,
64,
11, ('XavierFill', {}), ('ConstantFill', {}),
stride=4,
pad=2)
relu1 = brew.relu(model, conv1, "conv1")
pool1 = brew.max_pool(model, relu1, "pool1", kernel=3, stride=2)
conv2 = brew.conv(model,
pool1,
"conv2",
64,
192,
5, ('XavierFill', {}), ('ConstantFill', {}),
pad=2)
relu2 = brew.relu(model, conv2, "conv2")
pool2 = brew.max_pool(model, relu2, "pool2", kernel=3, stride=2)
conv3 = brew.conv(model,
pool2,
"conv3",
192,
384,
3, ('XavierFill', {}), ('ConstantFill', {}),
pad=1)
relu3 = brew.relu(model, conv3, "conv3")
conv4 = brew.conv(model,
relu3,
"conv4",
384,
256,
3, ('XavierFill', {}), ('ConstantFill', {}),
pad=1)
relu4 = brew.relu(model, conv4, "conv4")
conv5 = brew.conv(model,
relu4,
"conv5",
256,
256,
3, ('XavierFill', {}), ('ConstantFill', {}),
pad=1)
relu5 = brew.relu(model, conv5, "conv5")
pool5 = brew.max_pool(model, relu5, "pool5", kernel=3, stride=2)
fc6 = brew.fc(model, pool5, "fc6", 256 * 6 * 6, 4096, ('XavierFill', {}),
('ConstantFill', {}))
relu6 = brew.relu(model, fc6, "fc6")
fc7 = brew.fc(model, relu6, "fc7", 4096, 4096, ('XavierFill', {}),
('ConstantFill', {}))
relu7 = brew.relu(model, fc7, "fc7")
fc8 = brew.fc(model, relu7, "fc8", 4096, 1000, ('XavierFill', {}),
('ConstantFill', {}))
pred = brew.softmax(model, fc8, "pred")
xent = model.LabelCrossEntropy([pred, "label"], "xent")
loss = model.AveragedLoss(xent, "loss")
return model, 224
def test_multi_lstm(
self,
input_length,
dim_in,
max_num_units,
num_layers,
batch_size,
):
model = ModelHelper(name='external')
(
input_sequence,
seq_lengths,
) = model.net.AddExternalInputs(
'input_sequence',
'seq_lengths',
)
dim_out = [
np.random.randint(1, max_num_units + 1) for _ in range(num_layers)
]
h_all, h_last, c_all, c_last = rnn_cell.LSTM(
model=model,
input_blob=input_sequence,
seq_lengths=seq_lengths,
initial_states=None,
dim_in=dim_in,
dim_out=dim_out,
scope='test',
outputs_with_grads=(0, ),
return_params=False,
memory_optimization=False,
forget_bias=0.0,
forward_only=False,
return_last_layer_only=True,
)
workspace.RunNetOnce(model.param_init_net)
seq_lengths_val = np.random.randint(
1,
input_length + 1,
size=(batch_size),
).astype(np.int32)
input_sequence_val = np.random.randn(
input_length,
batch_size,
dim_in,
).astype(np.float32)
workspace.FeedBlob(seq_lengths, seq_lengths_val)
workspace.FeedBlob(input_sequence, input_sequence_val)
hidden_input_list = []
cell_input_list = []
i2h_w_list = []
i2h_b_list = []
gates_w_list = []
gates_b_list = []
for i in range(num_layers):
hidden_input_list.append(
workspace.FetchBlob(
'test/initial_hidden_state_{}'.format(i)), )
cell_input_list.append(
workspace.FetchBlob('test/initial_cell_state_{}'.format(i)), )
i2h_w_list.append(
workspace.FetchBlob('test/layer_{}/i2h_w'.format(i)), )
i2h_b_list.append(
workspace.FetchBlob('test/layer_{}/i2h_b'.format(i)), )
gates_w_list.append(
workspace.FetchBlob('test/layer_{}/gates_t_w'.format(i)), )
gates_b_list.append(
workspace.FetchBlob('test/layer_{}/gates_t_b'.format(i)), )
workspace.RunNetOnce(model.net)
h_all_calc = workspace.FetchBlob(h_all)
h_last_calc = workspace.FetchBlob(h_last)
c_all_calc = workspace.FetchBlob(c_all)
c_last_calc = workspace.FetchBlob(c_last)
h_all_ref, h_last_ref, c_all_ref, c_last_ref = multi_lstm_reference(
input_sequence_val,
hidden_input_list,
cell_input_list,
i2h_w_list,
i2h_b_list,
gates_w_list,
gates_b_list,
seq_lengths_val,
forget_bias=0.0,
)
h_all_delta = np.abs(h_all_ref - h_all_calc).sum()
h_last_delta = np.abs(h_last_ref - h_last_calc).sum()
c_all_delta = np.abs(c_all_ref - c_all_calc).sum()
c_last_delta = np.abs(c_last_ref - c_last_calc).sum()
self.assertAlmostEqual(h_all_delta, 0.0, places=5)
self.assertAlmostEqual(h_last_delta, 0.0, places=5)
self.assertAlmostEqual(c_all_delta, 0.0, places=5)
self.assertAlmostEqual(c_last_delta, 0.0, places=5)
input_values = {
'input_sequence': input_sequence_val,
'seq_lengths': seq_lengths_val,
}
for param in model.GetParams():
value = workspace.FetchBlob(param)
input_values[str(param)] = value
output_sum = model.net.SumElements(
[h_all],
'output_sum',
average=True,
)
fake_loss = model.net.Tanh(output_sum, )
for param in model.GetParams():
gradient_checker.NetGradientChecker.Check(
model.net,
outputs_with_grad=[fake_loss],
input_values=input_values,
input_to_check=str(param),
print_net=False,
step_size=0.0001,
threshold=0.05,
)
def _prepare_attention(t,
n,
dim_in,
encoder_dim,
forward_only=False,
T=None,
dim_out=None,
residual=False):
if dim_out is None:
dim_out = [dim_in]
print("Dims: t={} n={} dim_in={} dim_out={}".format(t, n, dim_in, dim_out))
model = ModelHelper(name='external')
def generate_input_state(shape):
return np.random.random(shape).astype(np.float32)
initial_states = []
for layer_id, d in enumerate(dim_out):
h, c = model.net.AddExternalInputs(
"hidden_init_{}".format(layer_id),
"cell_init_{}".format(layer_id),
)
initial_states.extend([h, c])
workspace.FeedBlob(h, generate_input_state((1, n, d)))
workspace.FeedBlob(c, generate_input_state((1, n, d)))
awec_init = model.net.AddExternalInputs([
'initial_attention_weighted_encoder_context',
])
initial_states.append(awec_init)
workspace.FeedBlob(
awec_init,
generate_input_state((1, n, encoder_dim)),
)
# Due to convoluted RNN scoping logic we make sure that things
# work from a namescope
with scope.NameScope("test_name_scope"):
(
input_blob,
seq_lengths,
encoder_outputs,
weighted_encoder_outputs,
) = model.net.AddScopedExternalInputs(
'input_blob',
'seq_lengths',
'encoder_outputs',
'weighted_encoder_outputs',
)
layer_input_dim = dim_in
cells = []
for layer_id, d in enumerate(dim_out):
cell = rnn_cell.MILSTMCell(
name='decoder_{}'.format(layer_id),
forward_only=forward_only,
input_size=layer_input_dim,
hidden_size=d,
forget_bias=0.0,
memory_optimization=False,
)
cells.append(cell)
layer_input_dim = d
decoder_cell = rnn_cell.MultiRNNCell(
cells,
name='decoder',
residual_output_layers=range(1, len(cells)) if residual else None,
)
attention_cell = rnn_cell.AttentionCell(
encoder_output_dim=encoder_dim,
encoder_outputs=encoder_outputs,
decoder_cell=decoder_cell,
decoder_state_dim=dim_out[-1],
name='attention_decoder',
attention_type=AttentionType.Recurrent,
weighted_encoder_outputs=weighted_encoder_outputs,
attention_memory_optimization=True,
)
attention_cell = (attention_cell if T is None else
rnn_cell.UnrolledCell(attention_cell, T))
output_indices = decoder_cell.output_indices
output_indices.append(2 * len(cells))
outputs_with_grads = [2 * i for i in output_indices]
final_output, state_outputs = attention_cell.apply_over_sequence(
model=model,
inputs=input_blob,
seq_lengths=seq_lengths,
initial_states=initial_states,
outputs_with_grads=outputs_with_grads,
)
workspace.RunNetOnce(model.param_init_net)
workspace.FeedBlob(
seq_lengths,
np.random.randint(1, t + 1, size=(n, )).astype(np.int32))
return {
'final_output': final_output,
'net': model.net,
'initial_states': initial_states,
'input_blob': input_blob,
'encoder_outputs': encoder_outputs,
'weighted_encoder_outputs': weighted_encoder_outputs,
'outputs_with_grads': outputs_with_grads,
}
def _create_rnn(cls, init_model, pred_model, n, opset_version):
assert init_model is not None, "cannot convert RNNs without access to the full model"
assert pred_model is not None, "cannot convert RNNs without access to the full model"
attrs = dict(n.attrs) # make a copy, which is safe to mutate
hidden_size = attrs.pop('hidden_size')
activation = force_unicode(attrs.pop('activations', ('tanh', ))[0])
direction = force_unicode(attrs.pop('direction', 'forward'))
assert not attrs, "unsupported RNN attributes: " + str(attrs.keys())
assert direction in ['forward',
'bidirectional'], "unsupported backwards RNN"
input_blob, W, R, B, sequence_lens, initial_h = n.inputs
if sequence_lens == "":
sequence_lens = None
input_size = cls._rnn_shape_inference(init_model, pred_model, n,
input_blob, W)
if input_size is None:
raise RuntimeError(
"best-effort shape inference for RNN input failed")
init_net = core.Net("init-net")
pred_mh = ModelHelper()
def make_rnn(direction_offset):
name = cls.dummy_name()
# input and recurrence biases are squashed together in
# onnx but not in caffe2
bias_offset = 2 * direction_offset * hidden_size
init_net.Slice(B,
name + "/i2h_b",
starts=[bias_offset + 0 * hidden_size],
ends=[bias_offset + 1 * hidden_size])
init_net.Slice(B,
name + "/gates_t_b",
starts=[bias_offset + 1 * hidden_size],
ends=[bias_offset + 2 * hidden_size])
weight_offset = direction_offset * hidden_size
init_net.Slice(W,
name + '/i2h_w',
starts=[weight_offset + 0 * hidden_size, 0],
ends=[weight_offset + 1 * hidden_size, -1])
init_net.Slice(R,
name + '/gates_t_w',
starts=[weight_offset + 0 * hidden_size, 0],
ends=[weight_offset + 1 * hidden_size, -1])
initial_h_sliced = name + '/initial_h'
init_net.Slice(initial_h,
initial_h_sliced,
starts=[direction_offset + 0, 0, 0],
ends=[direction_offset + 1, -1, -1])
if direction_offset == 1:
input = pred_mh.net.ReversePackedSegs(
[input_blob, sequence_lens], name + "/input-reversed")
else:
input = input_blob
hidden_t_all, hidden_t_last = rnn_cell.BasicRNN(
pred_mh,
input,
sequence_lens, [initial_h_sliced],
input_size,
hidden_size,
name,
drop_states=False,
forward_only=True,
activation=activation)
if direction_offset == 1:
hidden_t_all = pred_mh.net.ReversePackedSegs(
[hidden_t_all, sequence_lens], name + "/output-reversed")
return hidden_t_all, hidden_t_last
if direction == 'forward':
hidden_t_all, hidden_t_last = make_rnn(0)
# in the forward case, storage is shared between the two
# outputs. We need to decouple them so that the
# VariableLengthSequencePadding only mutates n.outputs[0]
pred_mh.net.Copy(hidden_t_last, n.outputs[1])
pred_mh.net = pred_mh.net.Clone(
"dummy-clone-net", blob_remap={hidden_t_all: n.outputs[0]})
elif direction == 'bidirectional':
hidden_t_all_f, hidden_t_last_f = make_rnn(0)
hidden_t_all_b, hidden_t_last_b = make_rnn(1)
pred_mh.net.Concat([hidden_t_all_f, hidden_t_all_b],
[n.outputs[0], cls.dummy_name()],
axis=2)
pred_mh.net.Concat([hidden_t_last_f, hidden_t_last_b],
[n.outputs[1], cls.dummy_name()],
axis=0)
if sequence_lens is not None:
pred_mh.net.VariableLengthSequencePadding(
[n.outputs[0], sequence_lens], [n.outputs[0]])
return Caffe2Ops(list(pred_mh.Proto().op), list(init_net.Proto().op),
list(pred_mh.Proto().external_input))
def test_multiple_optimizers(self):
from caffe2.python import brew, core, optimizer
from caffe2.python.model_helper import ModelHelper
model = ModelHelper(name="test")
fc1 = brew.fc(model, 'data', 'fc1', 100, 50)
fc2 = brew.fc(model, fc1, 'fc2', 50, 25)
pred = brew.fc(model, fc2, 'fc3', 25, 10)
(softmax, loss) = model.SoftmaxWithLoss(
[pred, 'label'],
['softmax', 'loss'],
)
model.AddGradientOperators([loss])
param_to_device = optimizer._get_param_to_device(model)
def infer_blob_device(blob_name):
return optimizer.get_param_device(blob_name,
"{}_grad".format(blob_name),
param_to_device)
sgd_1 = optimizer.SgdOptimizer(base_learning_rate=0.1)
sgd_2 = optimizer.SgdOptimizer(base_learning_rate=0.2)
adagrad = optimizer.AdagradOptimizer()
# Check same optimizer share the same learning rate.
with core.DeviceScope(infer_blob_device("fc1_w")):
sgd_1(model.net, model.param_init_net, "fc1_w", "fc1_w_grad")
with core.DeviceScope(infer_blob_device("fc1_b")):
sgd_1(model.net, model.param_init_net, "fc1_b", "fc1_b_grad")
fc1_lr_blobs = []
for op in model.net.Proto().op:
if op.type == 'WeightedSum' and op.input[0] == 'fc1_w' or \
op.input[0] == 'fc1_b':
fc1_lr_blobs.append(op.input[3])
self.assertEqual(fc1_lr_blobs[0], fc1_lr_blobs[1])
# Check different instance of the same optimizer has a different lr.
with core.DeviceScope(infer_blob_device("fc2_w")):
sgd_2(model.net, model.param_init_net, "fc2_w", "fc2_w_grad")
with core.DeviceScope(infer_blob_device("fc2_b")):
sgd_2(model.net, model.param_init_net, "fc2_b", "fc2_b_grad")
fc2_lr_blobs = []
for op in model.net.Proto().op:
if op.type == 'WeightedSum' and op.input[0] == 'fc2_w' or \
op.input[0] == 'fc2_b':
self.assertTrue(op.input[3] not in fc1_lr_blobs)
fc2_lr_blobs.append(op.input[3])
self.assertEqual(fc2_lr_blobs[0], fc2_lr_blobs[1])
# Check different optimizer type case
with core.DeviceScope(infer_blob_device("fc3_w")):
adagrad(model.net, model.param_init_net, "fc3_w", "fc3_w_grad")
with core.DeviceScope(infer_blob_device("fc3_b")):
adagrad(model.net, model.param_init_net, "fc3_b", "fc3_b_grad")
fc3_lr_blobs = []
for op in model.net.Proto().op:
if op.type == 'Adagrad' and op.input[0] == 'fc3_w' or \
op.input[0] == 'fc3_b':
self.assertTrue(op.input[3] not in fc2_lr_blobs)
self.assertTrue(op.input[3] not in fc1_lr_blobs)
fc3_lr_blobs.append(op.input[3])
self.assertEqual(fc3_lr_blobs[0], fc3_lr_blobs[1])
def double_matmul():
model = ModelHelper(name="r")
fc0 = brew.fc(model, "data", "fc0", 10, 10)
fc1 = brew.fc(model, fc0, "fc1", 10, 10)
model.Proto().external_output[:] = [str(fc0), str(fc1)]
return model, [(1, 10)]
def simple_relu():
model = ModelHelper(name="r")
brew.relu(model, "data", "fc")
return model, [(1, 10)]
def complex_resnet():
model = ModelHelper(name="r", arg_scope={"order": "NCHW", "is_test": True})
resnet.create_resnet50(
model, "data", num_input_channels=1, num_labels=5, is_test=True,
no_loss=True)
return model, [(1, 1, 224, 224)]
def simple_fc():
model = ModelHelper(name="r")
brew.fc(model, "data", "fc", 10, 10)
return model, [(1, 10)]
def simple_resnet():
model = ModelHelper(name="r", arg_scope={"order": "NCHW", "is_test": True})
resnet.create_resnet_32x32(
model, "data", num_input_channels=1, num_groups=1, num_labels=5,
is_test=True)
return model, [(1, 1, 32, 32)]
def simple_cnn():
model = ModelHelper(name="r", arg_scope={"order": "NCHW", "is_test": True})
brew.conv(model, "data", 'conv1', 3, 16, kernel=3, stride=1)
brew.spatial_bn(model, 'conv1', 'conv1_spatbn', 16, epsilon=1e-3)
brew.relu(model, 'conv1_spatbn', 'relu1')
return model, [(1, 3, 32, 32)]
def main(args):
# User defined keyword arguments
kwargs = {"order": "NCHW"}
kwargs.update(dict(args.kwargs))
model = ModelHelper(name=args.benchmark_name)
op_type = args.operator # assumes a brew type op name
input_name = args.input_name
output_name = args.output_name
iters = int(args.iters)
for i in range(iters):
input_blob_name = input_name + (str(i) if i > 0 and args.chain else '')
output_blob_name = output_name + str(i + 1)
add_op = getattr(brew, op_type)
add_op(model, input_blob_name, output_blob_name, **kwargs)
if args.chain:
input_name, output_name = output_name, input_name
workspace.RunNetOnce(model.param_init_net)
extra_init_net_ops = []
def make_blob_on_context(blob_name, blob_data, context):
if context.upper() != "CPU":
blob_name_modified = "{}_CPU".format(blob_name)
else: # CPU case is simple
blob_name_modified = blob_name
fill_op = core.CreateOperator(
"GivenTensorFill", [], [blob_name_modified],
arg=[
utils.MakeArgument("shape", blob_data.shape),
utils.MakeArgument("values", blob_data)
])
extra_init_net_ops.append(fill_op)
# We need to create CPU blobs and add some copy operations in
# the init_net
if context.upper() == "OPENGL":
copy_op = core.CreateOperator("CopyToOpenGL", [blob_name_modified],
[blob_name])
extra_init_net_ops.append(copy_op)
for unparsed_blob in args.blob:
name, unparsed_dims = unparsed_blob.split('=')
dims = [int(d) for d in unparsed_dims.split(',')]
np_input = np.random.rand(*dims).astype(np.float32)
make_blob_on_context(name, np_input, args.context)
init_net, predict_net = mobile_exporter.Export(workspace, model.net,
model.params)
init_net.op.extend(extra_init_net_ops)
# Handle manual rewrite
if args.context.upper() == "OPENGL":
old_ops = [op for op in predict_net.op]
del predict_net.op[:]
for op in old_ops:
op.type = 'OpenGL{}'.format(op.type)
predict_net.op.extend(old_ops)
if args.debug:
print("init_net:")
for op in init_net.op:
print(" ", op.type, op.input, "-->", op.output)
print("predict_net:")
for op in predict_net.op:
print(" ", op.type, op.input, "-->", op.output)
with open(args.predict_net, 'wb') as f:
f.write(predict_net.SerializeToString())
with open(args.init_net, 'wb') as f:
f.write(init_net.SerializeToString())
def _create_rnn_variant(cls, init_model, pred_model, n, opset_version):
assert init_model is not None, "cannot convert RNNs without access to the full model"
assert pred_model is not None, "cannot convert RNNs without access to the full model"
attrs = dict(n.attrs) # make a copy, which is safe to mutate
hidden_size = attrs.pop('hidden_size')
direction = force_unicode(attrs.pop('direction', 'forward'))
if n.op_type == 'RNN':
activation = force_unicode(attrs.pop('activations', ('tanh', ))[0])
elif n.op_type == 'GRU':
linear_before_reset = attrs.pop('linear_before_reset', 0)
assert not attrs, "unsupported RNN attributes: " + str(attrs.keys())
assert direction in ['forward', 'bidirectional'
], "unsupported backwards RNN/GRU/LSTM"
if n.op_type in ['RNN', 'GRU']:
input_blob, W, R, B, sequence_lens, initial_h = n.inputs
elif n.op_type == 'LSTM':
input_blob, W, R, B, sequence_lens, initial_h, initial_c = n.inputs
if sequence_lens == "":
sequence_lens = None
for x in itertools.chain(init_model.graph.input,
init_model.graph.value_info,
pred_model.graph.input,
pred_model.graph.value_info):
if x.name == W:
input_size = x.type.tensor_type.shape.dim[2].dim_value
break
else:
raise RuntimeError(
"best-effort shape inference for RNN/GRU/LSTM failed")
pred_mh = ModelHelper()
init_net = core.Net("init-net")
init_net.Reshape(W, [W, cls.dummy_name()], shape=[1, -1, 0])
init_net.Squeeze(W, W, dims=[0])
init_net.Reshape(R, [R, cls.dummy_name()], shape=[1, -1, 0])
init_net.Squeeze(R, R, dims=[0])
init_net.Reshape(B, [B, cls.dummy_name()], shape=[1, -1])
init_net.Squeeze(B, B, dims=[0])
if n.op_type == 'RNN':
def reform(*args):
pass
def make_cell(*args, **kwargs):
return rnn_cell.BasicRNN(*args,
activation=activation,
**kwargs)
def make_rnn(direction_offset):
return cls._make_rnn_direction(
input_blob, B, W, R, [(initial_h, '/initial_h')],
sequence_lens, pred_mh, init_net, input_size, hidden_size,
1, direction_offset, "/i2h_b", "/gates_t_b", "/i2h_w",
"/gates_t_w", reform, make_cell, lambda x: x)
elif n.op_type == 'GRU':
def reform(Bi, Br, W_, R_, name, hidden_size, init_net):
# caffe2 has a different order from onnx. We need to rearrange
# z r h -> r z h
reforms = ((W_, 'i2h_w', True, [
(0, -1)
]), (R_, 'gate_t_w', False, [
(0, -1)
]), (Bi, 'i2h_b', True, []), (Br, 'gate_t_b', False, []))
cls._rnn_reform_weights(reforms, name, hidden_size, init_net,
['update', 'reset', 'output'],
[1, 0, 2])
def make_cell(*args, **kwargs):
return gru_cell.GRU(*args,
linear_before_reset=linear_before_reset,
**kwargs)
def make_rnn(direction_offset):
return cls._make_rnn_direction(
input_blob, B, W, R, [(initial_h, '/initial_h')],
sequence_lens, pred_mh, init_net, input_size, hidden_size,
3, direction_offset, "_bias_i2h", "_bias_gates",
"/i2h_w_pre", "/gates_t_w_pre", reform, make_cell,
lambda x: x)
elif n.op_type == 'LSTM':
def reform(Bi, Br, W_, R_, name, hidden_size, init_net):
# caffe2 has a different order from onnx. We need to rearrange
# i o f c -> i f o c
reforms = ((W_, 'i2h_w', True, [
(0, -1)
]), (R_, 'gates_t_w', True, [
(0, -1)
]), (Bi, 'i2h_b', True, []), (Br, 'gates_t_b', True, []))
cls._rnn_reform_weights(reforms, name, hidden_size, init_net,
['input', 'output', 'forget', 'cell'],
[0, 2, 1, 3])
def make_cell(*args, **kwargs):
return rnn_cell.LSTM(*args, **kwargs)
def make_rnn(direction_offset):
return cls._make_rnn_direction(
input_blob, B, W, R, [(initial_h, '/initial_h'),
(initial_c, '/initial_c')],
sequence_lens, pred_mh, init_net, input_size, hidden_size,
4, direction_offset, "/i2h_b", "/gates_t_b", "/i2h_w",
"/gates_t_w", reform, make_cell,
lambda x: [x[0], x[1], x[3]])
if direction == 'forward':
outputs = make_rnn(0)
# in the forward case, storage is shared between the
# last outputs. We need to decouple them so that the
# VariableLengthSequencePadding only mutates
# n.outputs[0]
for i in range(1, len(outputs)):
pred_mh.net.Copy(outputs[i], n.outputs[i])
if sequence_lens is not None:
pred_mh.net.VariableLengthSequencePadding(
[outputs[0], sequence_lens], [outputs[0]])
pred_mh.net.ExpandDims([outputs[0]], [n.outputs[0]], dims=[1])
elif direction == 'bidirectional':
outputs_f = make_rnn(0)
outputs_b = make_rnn(1)
concatted_output, _ = pred_mh.net.Concat(
[outputs_f[0], outputs_b[0]],
[cls.dummy_name(), cls.dummy_name()],
axis=2)
if sequence_lens is not None:
pred_mh.net.VariableLengthSequencePadding(
[concatted_output, sequence_lens], [concatted_output])
reshaped_output, _ = pred_mh.net.Reshape(
concatted_output,
[cls.dummy_name(), cls.dummy_name()],
shape=[0, 0, -1, 2])
pred_mh.net.Transpose(reshaped_output,
n.outputs[0],
axes=[0, 3, 1, 2])
for i in range(1, len(n.outputs)):
pred_mh.net.Concat(
[outputs_f[i], outputs_b[i]],
[n.outputs[i], cls.dummy_name()],
axis=0)
# We want to decide whether to put all of our weight-reshaping
# operators in the init net or the predict net. We can put
# them in the init net iff the inputs to those operators are
# already available, either as graph initializers, or as the
# output of other operators in the init net. The latter case
# occurs, for example, when exporting from pytorch to onnx.
# In most production use, we expect has_initializers to be
# true.
initializers = {i.name for i in init_model.graph.initializer}
outputs = {
output
for node in init_model.graph.node for output in node.output
}
has_initializers = all(x in initializers or x in outputs
for x in (W, R, B))
pred_ops = []
init_ops = []
(init_ops if has_initializers else pred_ops).extend(
init_net.Proto().op)
pred_ops.extend(pred_mh.Proto().op)
return Caffe2Ops(pred_ops, init_ops,
list(pred_mh.Proto().external_input))
def alexnet():
model = ModelHelper(name="r", arg_scope={"order": "NCHW", "is_test": True})
conv1 = brew.conv(
model,
"data",
"conv1",
3,
64,
11, ('XavierFill', {}), ('ConstantFill', {}),
stride=4,
pad=2
)
relu1 = brew.relu(model, conv1, "conv1")
pool1 = brew.max_pool(model, relu1, "pool1", kernel=3, stride=2, pad=0,
legacy_pad=3)
lrn1 = brew.lrn(
model, pool1, "pool1_lrn", size=5, alpha=1.0e-4, beta=0.75, bias=1.0)
conv2 = brew.conv(
model,
lrn1,
"conv2",
64,
192,
5,
('XavierFill', {}),
('ConstantFill', {}),
pad=2
)
relu2 = brew.relu(model, conv2, "conv2")
pool2 = brew.max_pool(model, relu2, "pool2", kernel=3, stride=2)
lrn2 = brew.lrn(
model, pool2, "pool2_lrn", size=5, alpha=1.0e-4, beta=0.75, bias=1.0)
conv3 = brew.conv(
model,
lrn2,
"conv3",
192,
384,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1
)
relu3 = brew.relu(model, conv3, "conv3")
conv4 = brew.conv(
model,
relu3,
"conv4",
384,
256,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1
)
relu4 = brew.relu(model, conv4, "conv4")
conv5 = brew.conv(
model,
relu4,
"conv5",
256,
256,
3,
('XavierFill', {}),
('ConstantFill', {}),
pad=1
)
relu5 = brew.relu(model, conv5, "conv5")
pool5 = brew.max_pool(model, relu5, "pool5", kernel=3, stride=2)
fc6 = brew.fc(
model,
pool5, "fc6", 256 * 6 * 6, 4096, ('XavierFill', {}),
('ConstantFill', {})
)
relu6 = brew.relu(model, fc6, "fc6")
fc7 = brew.fc(
model, relu6, "fc7", 4096, 4096, ('XavierFill', {}), ('ConstantFill', {})
)
relu7 = brew.relu(model, fc7, "fc7")
drop7 = brew.dropout(model, relu7, "fc7_dropout", is_test=1, ratio=0.5)
fc8 = brew.fc(
model, drop7, "fc8", 4096, 1000, ('XavierFill', {}), ('ConstantFill', {})
)
relu8 = brew.relu(model, fc8, "fc8")
brew.dropout(model, relu8, "fc8_dropout", is_test=1, ratio=0.5)
return model, [(1, 3, 224, 224)]
from __future__ import unicode_literals
from caffe2.python import brew, core, scope, workspace
from caffe2.python.modeling.parameter_info import ParameterTags
from caffe2.python.model_helper import ModelHelper
from caffe2.python.cnn import CNNModelHelper
import unittest
import numpy as np
m, k, n = (1, 28 * 28, 10) # [m][k] * [k][n] = [m][n]
x = np.random.rand(m, k).astype(np.float32) - 0.5 # x = m*k 2D tensor
workspace.ResetWorkspace() # clear workspace
workspace.FeedBlob("x", x) # feed x as a blob
model = ModelHelper(name="test_model") # create model
model.Proto() # print model's protocol buffer before add operator
brew.fc(
model, "x", "y", k, n
) # fully connected NN, weight = k*n 2D tensor /// bias, y = m*n 2D tensor
brew.softmax(model, "y", "z")
model.Validate()
model.Proto() # print model's protocol buffer after add operator
workspace.RunNetOnce(
model.param_init_net) # init [y_w(weight), y_b(bias) (randomize)]
# weight is 2D array, bias is 1D array
workspace.Blobs() # print workspace's blobs
# workspace.FetchBlob("y_w")
# workspace.FetchBlob("y_b")
def test_basic_rnn(self, seed, seq_length, batch_size, input_size,
hidden_size, drop_states, sequence_lengths, gc, dc):
np.random.seed(seed)
seq_lengths_data = np.random.randint(1,
seq_length + 1,
size=(batch_size, )).astype(
np.int32)
input_blob_data = np.random.randn(seq_length, batch_size,
input_size).astype(np.float32)
initial_h_data = np.random.randn(batch_size,
hidden_size).astype(np.float32)
gates_t_w_data = np.random.randn(hidden_size,
hidden_size).astype(np.float32)
gates_t_b_data = np.random.randn(hidden_size).astype(np.float32)
i2h_w_data = np.random.randn(hidden_size,
input_size).astype(np.float32)
i2h_b_data = np.random.randn(hidden_size).astype(np.float32)
with core.DeviceScope(gc):
with hu.temp_workspace():
workspace.FeedBlob('input_blob',
input_blob_data,
device_option=gc)
workspace.FeedBlob('seq_lengths',
seq_lengths_data,
device_option=gc)
workspace.FeedBlob('initial_h',
initial_h_data,
device_option=gc)
workspace.FeedBlob('basic_rnn/gates_t_w',
gates_t_w_data,
device_option=gc)
workspace.FeedBlob('basic_rnn/gates_t_b',
gates_t_b_data,
device_option=gc)
workspace.FeedBlob('basic_rnn/i2h_w',
i2h_w_data,
device_option=gc)
workspace.FeedBlob('basic_rnn/i2h_b',
i2h_b_data,
device_option=gc)
model = ModelHelper(name='model')
hidden_t_all, _ = rnn_cell.BasicRNN(
model,
'input_blob',
'seq_lengths' if sequence_lengths else None, ['initial_h'],
input_size,
hidden_size,
"basic_rnn",
activation='tanh',
forward_only=True,
drop_states=drop_states)
workspace.RunNetOnce(model.net)
result = workspace.FetchBlob(hidden_t_all)
reference = basic_rnn_reference(
input_blob_data,
initial_h_data,
i2h_w_data,
i2h_b_data,
gates_t_w_data,
gates_t_b_data,
seq_lengths_data if sequence_lengths else None,
drop_states=drop_states,
use_sequence_lengths=sequence_lengths)
np.testing.assert_allclose(result, reference, atol=1e-4, rtol=1e-4)
def _create_gru(cls, init_model, pred_model, n, opset_version):
assert init_model is not None, "cannot convert GRUs without access to the full model"
assert pred_model is not None, "cannot convert GRUs without access to the full model"
attrs = dict(n.attrs) # make a copy, which is safe to mutate
hidden_size = attrs.pop('hidden_size')
linear_before_reset = attrs.pop('linear_before_reset', 0)
direction = force_unicode(attrs.pop('direction', 'forward'))
assert not attrs, "unsupported GRU attributes: " + str(attrs.keys())
assert direction in ['forward',
'bidirectional'], "unsupported backwards GRU"
input_blob, W, R, B, sequence_lens, initial_h = n.inputs
if sequence_lens == "":
sequence_lens = None
input_size = cls._rnn_shape_inference(init_model, pred_model, n,
input_blob, W)
if input_size is None:
raise RuntimeError(
"best-effort shape inference for GRU input failed")
init_net = core.Net("init-net")
pred_mh = ModelHelper()
def make_gru(direction_offset):
name = cls.dummy_name()
# input and recurrence biases are squashed together in
# onnx but not in caffe2
bias_offset = 6 * direction_offset * hidden_size
Bi = init_net.Slice(B,
name + "_bias_i2h",
starts=[bias_offset + 0 * hidden_size],
ends=[bias_offset + 3 * hidden_size])
Br = init_net.Slice(B,
name + "_bias_gates",
starts=[bias_offset + 3 * hidden_size],
ends=[bias_offset + 6 * hidden_size])
weight_offset = 3 * direction_offset * hidden_size
W_ = init_net.Slice(W,
name + '/i2h_w_pre',
starts=[weight_offset + 0 * hidden_size, 0],
ends=[weight_offset + 3 * hidden_size, -1])
R_ = init_net.Slice(R,
name + '/gates_t_w_pre',
starts=[weight_offset + 0 * hidden_size, 0],
ends=[weight_offset + 3 * hidden_size, -1])
# caffe2 has a different order from onnx. We need to rearrange
# z r h -> r z h
reforms = ((W_, 'i2h_w', True, [(0, -1)]), (R_, 'gate_t_w', False,
[(0, -1)]),
(Bi, 'i2h_b', True, []), (Br, 'gate_t_b', False, []))
for name_from, name_to, do_concat, extra_dims in reforms:
xz, xr, xh = [
'%s/%s_%s' % (name, prefix, name_to)
for prefix in ('update', 'reset', 'output')
]
for i, x in enumerate([xz, xr, xh]):
dim0 = i * hidden_size, (i + 1) * hidden_size
starts, ends = zip(dim0, *extra_dims)
init_net.Slice(name_from, x, starts=starts, ends=ends)
if do_concat:
init_net.Concat(
[xr, xz, xh],
['%s/%s' % (name, name_to),
cls.dummy_name()],
axis=0)
initial_h_sliced = name + '/initial_h'
init_net.Slice(initial_h,
initial_h_sliced,
starts=[direction_offset + 0, 0, 0],
ends=[direction_offset + 1, -1, -1])
if direction_offset == 1:
input = pred_mh.net.ReversePackedSegs(
[input_blob, sequence_lens], name + "/input-reversed")
else:
input = input_blob
hidden_t_all, hidden_t_last = gru_cell.GRU(
pred_mh,
input,
sequence_lens, [initial_h_sliced],
input_size,
hidden_size,
name,
drop_states=False,
forward_only=True,
linear_before_reset=linear_before_reset)
if direction_offset == 1:
hidden_t_all = pred_mh.net.ReversePackedSegs(
[hidden_t_all, sequence_lens], name + "/output-reversed")
return hidden_t_all, hidden_t_last
if direction == 'forward':
hidden_t_all, hidden_t_last = make_gru(0)
# in the forward case, storage is shared between the two
# outputs. We need to decouple them so that the
# VariableLengthSequencePadding only mutates n.outputs[0]
pred_mh.net.Copy(hidden_t_last, n.outputs[1])
pred_mh.net = pred_mh.net.Clone(
"dummy-clone-net", blob_remap={hidden_t_all: n.outputs[0]})
elif direction == 'bidirectional':
hidden_t_all_f, hidden_t_last_f = make_gru(0)
hidden_t_all_b, hidden_t_last_b = make_gru(1)
pred_mh.net.Concat([hidden_t_all_f, hidden_t_all_b],
[n.outputs[0], cls.dummy_name()],
axis=2)
pred_mh.net.Concat([hidden_t_last_f, hidden_t_last_b],
[n.outputs[1], cls.dummy_name()],
axis=0)
if sequence_lens is not None:
pred_mh.net.VariableLengthSequencePadding(
[n.outputs[0], sequence_lens], [n.outputs[0]])
return Caffe2Ops(list(pred_mh.Proto().op), list(init_net.Proto().op),
list(pred_mh.Proto().external_input))
def _create_gru(cls, init_model, pred_model, n, opset_version):
assert init_model is not None, "cannot convert GRUs without access to the full model"
assert pred_model is not None, "cannot convert GRUs without access to the full model"
attrs = dict(n.attrs) # make a copy, which is safe to mutate
hidden_size = attrs.pop('hidden_size')
linear_before_reset = attrs.pop('linear_before_reset')
assert not attrs, "unsupported GRU attributes: " + str(attrs.keys())
input_blob, W, R, B, sequence_lens, initial_h = n.inputs
input_size = cls._rnn_shape_inference(init_model, pred_model, n, input_blob, W)
if input_size is None:
raise RuntimeError("best-effort shape inference for GRU input failed")
name = dummy_name()
init_net = core.Net("init-net")
pred_mh = ModelHelper()
hidden_t_all, hidden_t_last = gru_cell.GRU(
pred_mh,
input_blob,
sequence_lens,
[initial_h],
input_size,
hidden_size,
name,
forward_only=True,
linear_before_reset=linear_before_reset
)
# input and recurrence biases are squashed together in onnx but not in caffe2
Bi = name + "_bias_i2h"
Br = name + "_bias_gates"
init_net.Slice(B, Bi, starts=[0*hidden_size], ends=[3*hidden_size])
init_net.Slice(B, Br, starts=[3*hidden_size], ends=[6*hidden_size])
# caffe2 has a different order from onnx. We need to rearrange
# z r h -> r z h
#
# TODO implement support for return_params in gru_cell.GRU.
# Until then, hardcode blob names.
reforms = ((W, 'i2h_w', True, [(0,input_size)]),
(R, 'gate_t_w', False, [(0,hidden_size)]),
(Bi, 'i2h_b', True, []),
(Br, 'gate_t_b', False, []))
for name_from, name_to, do_concat, extra_dims in reforms:
xz, xr, xh = ['%s/%s_%s' % (name, prefix, name_to) for prefix in ('update', 'reset', 'output')]
for i, x in enumerate([xz, xr, xh]):
dim0 = i * hidden_size, (i+1) * hidden_size
starts, ends = zip(dim0, *extra_dims)
init_net.Slice(name_from, x, starts=starts, ends=ends)
if do_concat:
init_net.Concat([xr, xz, xh], ['%s/%s' % (name, name_to), dummy_name()], axis=0)
pred_mh.net = pred_mh.net.Clone(
"dummy-clone-net",
blob_remap={ hidden_t_all: n.outputs[0], hidden_t_last: n.outputs[1] }
)
return Caffe2Ops(list(pred_mh.Proto().op),
list(init_net.Proto().op),
list(pred_mh.Proto().external_input))
def test_convolution_sync(self, net_type, num_workers, engine, gc, dc):
m = ModelHelper(name="test_model")
n = 1
d = 2
depth = 3
iters = 5
h = 5
w = 5
workspace.ResetWorkspace()
use_cudnn = engine == "CUDNN"
np.random.seed(1701)
# Build a binary tree of conv layers, summing at each node.
for i in reversed(range(depth)):
for j in range(2 ** i):
bottom_1 = "{}_{}".format(i + 1, 2 * j)
bottom_2 = "{}_{}".format(i + 1, 2 * j + 1)
mid_1 = "{}_{}_m".format(i + 1, 2 * j)
mid_2 = "{}_{}_m".format(i + 1, 2 * j + 1)
top = "{}_{}".format(i, j)
w1, b1, w2, b2 = np.random.randn(4).tolist()
brew.conv(
m,
bottom_1,
mid_1,
dim_in=d,
dim_out=d,
kernel=3,
weight_init=("ConstantFill", {"value": w1}),
bias_init=("ConstantFill", {"value": b1}),
cudnn_state=np.random.randint(0, 3),
stride=1,
pad=1,
deterministic=1,
use_cudnn=use_cudnn,
engine=engine,
)
brew.conv(
m,
bottom_2,
mid_2,
dim_in=d,
dim_out=d,
kernel=3,
stride=1,
pad=1,
weight_init=("ConstantFill", {"value": w2}),
bias_init=("ConstantFill", {"value": b2}),
deterministic=1,
cudnn_state=np.random.randint(0, 3),
use_cudnn=use_cudnn,
engine=engine,
)
m.net.Sum([mid_1, mid_2], top)
m.net.Flatten(["0_0"], ["0_0_flat"])
m.net.SquaredL2Distance(["0_0_flat", "label"], "xent")
m.net.AveragedLoss("xent", "loss")
input_to_grad = m.AddGradientOperators(["loss"])
m.Proto().device_option.CopyFrom(gc)
m.param_init_net.Proto().device_option.CopyFrom(gc)
m.Proto().type = net_type
m.Proto().num_workers = num_workers
self.ws.run(m.param_init_net)
def run():
import numpy as np
np.random.seed(1701)
input_blobs = ["{}_{}".format(depth, j) for j in range(2 ** depth)]
for input_blob in input_blobs:
self.ws.create_blob(input_blob).feed(
np.random.randn(n, d, h, w).astype(np.float32), device_option=gc
)
self.ws.create_blob("label").feed(
np.random.randn(n, d * h * w).astype(np.float32), device_option=gc
)
self.ws.run(m.net)
gradients = [
self.ws.blobs[str(input_to_grad[input_blob])].fetch()
for input_blob in input_blobs
]
return gradients
outputs = [run() for _ in range(iters)]
for output in outputs[1:]:
np.testing.assert_array_equal(outputs[0], output)
np.testing.assert_allclose(
np.sum(np.square(output)), 1763719461732352.0, rtol=1e-5
)
def _create_rnn_variant(cls, init_model, pred_model, n, opset_version):
assert init_model is not None, "cannot convert RNNs without access to the full model"
assert pred_model is not None, "cannot convert RNNs without access to the full model"
attrs = dict(n.attrs) # make a copy, which is safe to mutate
hidden_size = attrs.pop('hidden_size')
direction = force_unicode(attrs.pop('direction', 'forward'))
if n.op_type == 'RNN':
activation = force_unicode(attrs.pop('activations', ('tanh',))[0])
elif n.op_type == 'GRU':
linear_before_reset = attrs.pop('linear_before_reset', 0)
assert not attrs, "unsupported RNN attributes: " + str(attrs.keys())
assert direction in ['forward', 'bidirectional'], "unsupported backwards RNN/GRU/LSTM"
if n.op_type in ['RNN', 'GRU']:
input_blob, W, R, B, sequence_lens, initial_h = n.inputs
elif n.op_type == 'LSTM':
input_blob, W, R, B, sequence_lens, initial_h, initial_c = n.inputs
if sequence_lens == "":
sequence_lens = None
for x in itertools.chain(init_model.graph.input,
init_model.graph.value_info,
pred_model.graph.input,
pred_model.graph.value_info):
if x.name == W:
input_size = x.type.tensor_type.shape.dim[1].dim_value
break
else:
raise RuntimeError("best-effort shape inference for RNN/GRU/LSTM failed")
init_net = core.Net("init-net")
pred_mh = ModelHelper()
if n.op_type == 'RNN':
def reform(*args):
pass
def make_cell(*args, **kwargs):
return rnn_cell.BasicRNN(*args, activation=activation, **kwargs)
def make_rnn(direction_offset):
return cls._make_rnn_direction(
input_blob, B, W, R, [(initial_h, '/initial_h')], sequence_lens,
pred_mh, init_net, input_size, hidden_size, 1, direction_offset,
"/i2h_b", "/gates_t_b", "/i2h_w", "/gates_t_w",
reform, make_cell, lambda x: x)
elif n.op_type == 'GRU':
def reform(Bi, Br, W_, R_, name, hidden_size, init_net):
# caffe2 has a different order from onnx. We need to rearrange
# z r h -> r z h
reforms = ((W_, 'i2h_w', True, [(0,-1)]),
(R_, 'gate_t_w', False, [(0,-1)]),
(Bi, 'i2h_b', True, []),
(Br, 'gate_t_b', False, []))
cls._rnn_reform_weights(reforms, name, hidden_size, init_net,
['update', 'reset', 'output'], [1, 0, 2])
def make_cell(*args, **kwargs):
return gru_cell.GRU(*args, linear_before_reset=linear_before_reset, **kwargs)
def make_rnn(direction_offset):
return cls._make_rnn_direction(
input_blob, B, W, R, [(initial_h, '/initial_h')], sequence_lens,
pred_mh, init_net, input_size, hidden_size, 3, direction_offset,
"_bias_i2h", "_bias_gates", "/i2h_w_pre", "/gates_t_w_pre",
reform, make_cell, lambda x: x)
elif n.op_type == 'LSTM':
def reform(Bi, Br, W_, R_, name, hidden_size, init_net):
# caffe2 has a different order from onnx. We need to rearrange
# i o f c -> i f o c
reforms = ((W_, 'i2h_w', True, [(0, -1)]),
(R_, 'gates_t_w', True, [(0, -1)]),
(Bi, 'i2h_b' , True, []),
(Br, 'gates_t_b', True, []))
cls._rnn_reform_weights(reforms, name, hidden_size, init_net,
['input', 'output', 'forget', 'cell'], [0, 2, 1, 3])
def make_cell(*args, **kwargs):
return rnn_cell.LSTM(*args, **kwargs)
def make_rnn(direction_offset):
return cls._make_rnn_direction(
input_blob, B, W, R, [(initial_h, '/initial_h'), (initial_c, '/initial_c')], sequence_lens,
pred_mh, init_net, input_size, hidden_size, 4, direction_offset,
"/i2h_b", "/gates_t_b", "/i2h_w", "/gates_t_w",
reform, make_cell, lambda x: [x[0], x[1], x[3]])
if direction == 'forward':
outputs = make_rnn(0)
# in the forward case, storage is shared between the
# last outputs. We need to decouple them so that the
# VariableLengthSequencePadding only mutates
# n.outputs[0]
for i in range(1, len(outputs)):
pred_mh.net.Copy(outputs[i], n.outputs[i])
pred_mh.net = pred_mh.net.Clone(
"dummy-clone-net", blob_remap={ outputs[0]: n.outputs[0] }
)
elif direction == 'bidirectional':
outputs_f = make_rnn(0)
outputs_b = make_rnn(1)
pred_mh.net.Concat([outputs_f[0], outputs_b[0]],
[n.outputs[0], cls.dummy_name()], axis=2)
for i in range(1, len(n.outputs)):
pred_mh.net.Concat([outputs_f[i], outputs_b[i]],
[n.outputs[i], cls.dummy_name()], axis=0)
if sequence_lens is not None:
pred_mh.net.VariableLengthSequencePadding(
[n.outputs[0], sequence_lens], [n.outputs[0]])
return Caffe2Ops(list(pred_mh.Proto().op),
list(init_net.Proto().op),
list(pred_mh.Proto().external_input))
def _prepare_lstm(t,
n,
dim_in,
create_lstm,
outputs_with_grads,
forget_bias,
memory_optim=False,
forward_only=False,
drop_states=False,
T=None,
two_d_initial_states=None,
dim_out=None):
if dim_out is None:
dim_out = [dim_in]
print("Dims: ", t, n, dim_in, dim_out)
model = ModelHelper(name='external')
if two_d_initial_states is None:
two_d_initial_states = np.random.randint(2)
def generate_input_state(n, d):
if two_d_initial_states:
return np.random.randn(n, d).astype(np.float32)
return np.random.randn(1, n, d).astype(np.float32)
states = []
for layer_id, d in enumerate(dim_out):
h, c = model.net.AddExternalInputs(
"hidden_init_{}".format(layer_id),
"cell_init_{}".format(layer_id),
)
states.extend([h, c])
workspace.FeedBlob(h, generate_input_state(n, d).astype(np.float32))
workspace.FeedBlob(c, generate_input_state(n, d).astype(np.float32))
# Due to convoluted RNN scoping logic we make sure that things
# work from a namescope
with scope.NameScope("test_name_scope"):
input_blob, seq_lengths = model.net.AddScopedExternalInputs(
'input_blob', 'seq_lengths')
outputs = create_lstm(
model,
input_blob,
seq_lengths,
states,
dim_in=dim_in,
dim_out=dim_out,
scope="external/recurrent",
outputs_with_grads=outputs_with_grads,
memory_optimization=memory_optim,
forget_bias=forget_bias,
forward_only=forward_only,
drop_states=drop_states,
static_rnn_unroll_size=T,
)
workspace.RunNetOnce(model.param_init_net)
workspace.FeedBlob(
seq_lengths,
np.random.randint(1, t + 1, size=(n, )).astype(np.int32))
return outputs, model.net, states + [input_blob]
def _prepare_gru_unit_op(gc, n, d, outputs_with_grads,
forward_only=False, drop_states=False,
sequence_lengths=False,
two_d_initial_states=None):
print("Dims: (n,d) = ({},{})".format(n, d))
def generate_input_state(n, d):
if two_d_initial_states:
return np.random.randn(n, d).astype(np.float32)
else:
return np.random.randn(1, n, d).astype(np.float32)
model = ModelHelper(name='external')
with scope.NameScope("test_name_scope"):
if sequence_lengths:
hidden_t_prev, gates_t, seq_lengths, timestep = \
model.net.AddScopedExternalInputs(
"hidden_t_prev",
"gates_t",
'seq_lengths',
"timestep",
)
else:
hidden_t_prev, gates_t, timestep = \
model.net.AddScopedExternalInputs(
"hidden_t_prev",
"gates_t",
"timestep",
)
workspace.FeedBlob(
hidden_t_prev,
generate_input_state(n, d).astype(np.float32),
device_option=gc
)
workspace.FeedBlob(
gates_t,
generate_input_state(n, 3 * d).astype(np.float32),
device_option=gc
)
if sequence_lengths:
inputs = [hidden_t_prev, gates_t, seq_lengths, timestep]
else:
inputs = [hidden_t_prev, gates_t, timestep]
hidden_t = model.net.GRUUnit(
inputs,
['hidden_t'],
forget_bias=0.0,
drop_states=drop_states,
sequence_lengths=sequence_lengths,
)
model.net.AddExternalOutputs(hidden_t)
workspace.RunNetOnce(model.param_init_net)
if sequence_lengths:
# 10 is used as a magic number to simulate some reasonable timestep
# and generate some reasonable seq. lengths
workspace.FeedBlob(
seq_lengths,
np.random.randint(1, 10, size=(n,)).astype(np.int32),
device_option=gc
)
workspace.FeedBlob(
timestep,
np.random.randint(1, 10, size=(1,)).astype(np.int32),
device_option=core.DeviceOption(caffe2_pb2.CPU),
)
print("Feed {}".format(timestep))
return hidden_t, model.net
def test_lstm_extract_predictor_net(self):
model = ModelHelper(name="lstm_extract_test")
with core.DeviceScope(core.DeviceOption(caffe2_pb2.CPU, 0)):
output, _, _, _ = rnn_cell.LSTM(
model=model,
input_blob="input",
seq_lengths="seqlengths",
initial_states=("hidden_init", "cell_init"),
dim_in=20,
dim_out=40,
scope="test",
drop_states=True,
return_last_layer_only=True,
)
# Run param init net to get the shapes for all inputs
shapes = {}
workspace.RunNetOnce(model.param_init_net)
for b in workspace.Blobs():
shapes[b] = workspace.FetchBlob(b).shape
# But export in CPU
(predict_net, export_blobs) = ExtractPredictorNet(
net_proto=model.net.Proto(),
input_blobs=["input"],
output_blobs=[output],
device=core.DeviceOption(caffe2_pb2.CPU, 1),
)
# Create the net and run once to see it is valid
# Populate external inputs with correctly shaped random input
# and also ensure that the export_blobs was constructed correctly.
workspace.ResetWorkspace()
shapes['input'] = [10, 4, 20]
shapes['cell_init'] = [1, 4, 40]
shapes['hidden_init'] = [1, 4, 40]
print(predict_net.Proto().external_input)
self.assertTrue('seqlengths' in predict_net.Proto().external_input)
for einp in predict_net.Proto().external_input:
if einp == 'seqlengths':
workspace.FeedBlob("seqlengths",
np.array([10] * 4, dtype=np.int32))
else:
workspace.FeedBlob(
einp,
np.zeros(shapes[einp]).astype(np.float32),
)
if einp != 'input':
self.assertTrue(einp in export_blobs)
print(str(predict_net.Proto()))
self.assertTrue(workspace.CreateNet(predict_net.Proto()))
self.assertTrue(workspace.RunNet(predict_net.Proto().name))
# Validate device options set correctly for the RNNs
import google.protobuf.text_format as protobuftx
for op in predict_net.Proto().op:
if op.type == 'RecurrentNetwork':
for arg in op.arg:
if arg.name == "step_net":
step_proto = caffe2_pb2.NetDef()
protobuftx.Merge(arg.s.decode("ascii"), step_proto)
for step_op in step_proto.op:
self.assertEqual(0,
step_op.device_option.device_type)
self.assertEqual(1,
step_op.device_option.cuda_gpu_id)
elif arg.name == 'backward_step_net':
self.assertEqual(b"", arg.s)
def Inception(order, cudnn_ws, mkl):
my_arg_scope = {
'order': order,
'use_cudnn': True,
'cudnn_exhaustive_search': True,
'ws_nbytes_limit': str(cudnn_ws)
}
model = ModelHelper(name="inception", arg_scope=my_arg_scope)
conv1 = brew.conv(model,
"data",
"conv1",
3,
64,
7, ('XavierFill', {}), ('ConstantFill', {}),
stride=2,
pad=3)
relu1 = brew.relu(model, conv1, "conv1")
pool1 = brew.max_pool(model, relu1, "pool1", kernel=3, stride=2, pad=1)
conv2a = brew.conv(model, pool1, "conv2a", 64, 64, 1, ('XavierFill', {}),
('ConstantFill', {}))
conv2a = brew.relu(model, conv2a, conv2a)
conv2 = brew.conv(model,
conv2a,
"conv2",
64,
192,
3, ('XavierFill', {}), ('ConstantFill', {}),
pad=1)
relu2 = brew.relu(model, conv2, "conv2")
pool2 = brew.max_pool(model, relu2, "pool2", kernel=3, stride=2, pad=1)
# Inception modules
inc3 = _InceptionModule(model, pool2, 192, "inc3", 64, [96, 128], [16, 32],
32)
inc4 = _InceptionModule(model, inc3, 256, "inc4", 128, [128, 192],
[32, 96], 64)
pool5 = brew.max_pool(model, inc4, "pool5", kernel=3, stride=2, pad=1)
inc5 = _InceptionModule(model, pool5, 480, "inc5", 192, [96, 208],
[16, 48], 64)
inc6 = _InceptionModule(model, inc5, 512, "inc6", 160, [112, 224],
[24, 64], 64)
inc7 = _InceptionModule(model, inc6, 512, "inc7", 128, [128, 256],
[24, 64], 64)
inc8 = _InceptionModule(model, inc7, 512, "inc8", 112, [144, 288],
[32, 64], 64)
inc9 = _InceptionModule(model, inc8, 528, "inc9", 256, [160, 320],
[32, 128], 128)
pool9 = brew.max_pool(model, inc9, "pool9", kernel=3, stride=2, pad=1)
inc10 = _InceptionModule(model, pool9, 832, "inc10", 256, [160, 320],
[32, 128], 128)
inc11 = _InceptionModule(model, inc10, 832, "inc11", 384, [192, 384],
[48, 128], 128)
pool11 = brew.average_pool(model, inc11, "pool11", kernel=7, stride=1)
fc = brew.fc(model, pool11, "fc", 1024, 1000, ('XavierFill', {}),
('ConstantFill', {}))
# It seems that Soumith's benchmark does not have softmax on top
# for Inception. We will add it anyway so we can have a proper
# backward pass.
pred = brew.softmax(model, fc, "pred")
xent = model.LabelCrossEntropy([pred, "label"], "xent")
if not mkl:
loss = model.AveragedLoss(xent, "loss")
return model, 224
def test_stateful_convolution_forward_only(
self,
sequence_length,
conv_window,
batch_size,
state_size,
):
'''
This unit test demonstrates another ways of using RecurrentNetwork.
Imagine, that you want to compute convolution over a sequence,
but sequence elements are not given to you from the beginning,
so you have to loop over the sequence and compute convolution
for each element separately. This situation can occur,
during inference/generation step of the neural networks.
First of all, you have to provide actual input via recurrent states,
since the input of RecurrentNetwork should be known in advance.
Here, we use `fake_inputs` as the input,
and it's used by the op to extract batch size and sequence length.
The actual input sequence is stored in the recurrent state
`input_state`. At every step we generate a new element via input_state_t
(in this example, input_state_t is generated at random, but
in a real situation it can be created using convolution output
from the previous step).
A few important differences from regular RecurrentNetwork usecase:
1. input_state_t_prev is not only a single previous element of
input_state sequence. It is last conv_window elements including (!)
the current one - input_state_t. We specify that using `link_window`
argument of RecurrentNetwork. We need that many elements to
compute a single convolution step. Also, note that `link_window`
specifies how many elements to link starting at
`timestep` + `link_offset` position.
2. First few steps might require additional zero padding from the left,
since there is no enough element of input_state sequence are available.
So the initial_state for input_state contains several elements
(exactly how many pads we need for the first step). Also, because of
that all offseting over input_state sequnece is being shifted
by length of initial_input_state: see `link_offset` and `alias_offset`
arguments of RecurrentNetwork.
In this test, we assert that we get the same result
if we apply convolution over all elements simultaneously,
since the whole input_state sequence was generated at the end.
'''
model = ModelHelper(name='model')
fake_inputs = model.param_init_net.UniformFill(
[],
'fake_inputs',
min=-1.0,
max=1.0,
shape=[sequence_length, batch_size, state_size],
)
initial_input_state = model.param_init_net.ConstantFill(
[],
'initial_input_state',
value=0.0,
shape=[conv_window - 1, batch_size, state_size],
)
initial_output_state = model.param_init_net.ConstantFill(
[],
'initial_output_state',
value=0.0,
shape=[1, batch_size, state_size],
)
step_model = ModelHelper(name='step_model', param_model=model)
(
fake_input_t,
timestep,
input_state_t_prev,
) = step_model.net.AddExternalInputs(
'fake_input_t',
'timestep',
'input_state_t_prev',
)
conv_filter = step_model.param_init_net.XavierFill(
[],
'conv_filter',
shape=[state_size, 1, conv_window, state_size],
)
conv_bias = step_model.param_init_net.ConstantFill(
[],
'conv_bias',
shape=[state_size],
value=0.0,
)
step_model.params.extend([conv_filter, conv_bias])
input_state_t = step_model.net.UniformFill(
[],
'input_state_t',
min=-1.0,
max=1.0,
shape=[1, batch_size, state_size],
)
output_state_t = self._convolution_1d(
model=step_model,
inputs=input_state_t_prev,
conv_window=conv_window,
conv_filter=conv_filter,
conv_bias=conv_bias,
output_name='output_state_t',
left_pad=False,
)
initial_recurrent_states = [initial_input_state, initial_output_state]
all_inputs = (
[fake_inputs] + step_model.params + initial_recurrent_states
)
all_outputs = ['input_state_all', 'output_state_all']
recurrent_states = ['input_state', 'output_state']
input_state_all, output_state_all, _ = model.net.RecurrentNetwork(
all_inputs,
all_outputs + ['step_workspaces'],
param=[all_inputs.index(p) for p in step_model.params],
alias_src=recurrent_states,
alias_dst=all_outputs,
alias_offset=[conv_window - 1, 1],
recurrent_states=recurrent_states,
initial_recurrent_state_ids=[
all_inputs.index(s) for s in initial_recurrent_states
],
link_internal=[
str(input_state_t_prev),
str(input_state_t),
str(output_state_t),
],
link_external=['input_state', 'input_state', 'output_state'],
link_offset=[0, conv_window - 1, 1],
link_window=[conv_window, 1, 1],
backward_link_internal=[],
backward_link_external=[],
backward_link_offset=[],
step_net=str(step_model.net.Proto()),
backward_step_net='',
timestep='timestep' if timestep is None else str(timestep),
outputs_with_grads=[],
)
output_states_2 = self._convolution_1d(
model=model,
inputs=input_state_all,
conv_window=conv_window,
conv_filter=conv_filter,
conv_bias=conv_bias,
output_name='output_states_2',
left_pad=True,
)
workspace.RunNetOnce(model.param_init_net)
workspace.RunNetOnce(model.net)
np.testing.assert_almost_equal(
workspace.FetchBlob(output_state_all),
workspace.FetchBlob(output_states_2),
decimal=3,
)
def _create_lstm(cls, init_model, pred_model, n, opset_version):
assert init_model is not None, "cannot convert LSTMs without access to the full model"
assert pred_model is not None, "cannot convert LSTMs without access to the full model"
attrs = dict(n.attrs) # make a copy, which is safe to mutate
hidden_size = attrs.pop('hidden_size')
direction = force_unicode(attrs.pop('direction', 'forward'))
assert not attrs, "unsupported LSTM attributes: " + str(attrs.keys())
assert direction in ['forward', 'bidirectional'], "unsupported backwards LSTM"
input_blob, W, R, B, sequence_lens, initial_h, initial_c = n.inputs
if sequence_lens == "":
sequence_lens = None
input_size = cls._rnn_shape_inference(init_model, pred_model, n, input_blob, W)
if input_size is None:
raise RuntimeError("best-effort shape inference for LSTM input failed")
init_net = core.Net("init-net")
pred_mh = ModelHelper()
def make_lstm(direction_offset):
name = cls.dummy_name()
# input and recurrence biases are squashed together in
# onnx but not in caffe2
bias_offset = 8 * direction_offset * hidden_size
Bi = init_net.Slice(B, name + "_bias_i2h",
starts=[bias_offset + 0 * hidden_size],
ends =[bias_offset + 4 * hidden_size])
Br = init_net.Slice(B, name + "_bias_gates",
starts=[bias_offset + 4 * hidden_size],
ends =[bias_offset + 8 * hidden_size])
weight_offset = 4 * direction_offset * hidden_size
W_ = init_net.Slice(W, name + '/i2h_w_pre',
starts=[weight_offset + 0 * hidden_size, 0],
ends =[weight_offset + 4 * hidden_size,-1])
R_ = init_net.Slice(R, name + '/gates_t_w_pre',
starts=[weight_offset + 0 * hidden_size, 0],
ends =[weight_offset + 4 * hidden_size,-1])
# caffe2 has a different order from onnx. We need to rearrange
# i o f c -> i f o c
reforms = ((W_, 'i2h_w', [(0, -1)]),
(R_, 'gates_t_w', [(0, -1)]),
(Bi, 'i2h_b' , []),
(Br, 'gates_t_b', []))
for name_from, name_to, extra_dims in reforms:
xi, xo, xf, xc = [name_from + suffix for suffix in ("_i", "_o", "_f", "_c")]
for i, x in enumerate([xi, xo, xf, xc]):
dim0 = i * hidden_size, (i+1) * hidden_size
starts, ends = zip(dim0, *extra_dims)
init_net.Slice(name_from, x, starts=starts, ends=ends)
init_net.Concat([xi, xf, xo, xc], ['%s/%s' % (name, name_to), cls.dummy_name()], axis=0)
initial_h_sliced = name + '/initial_h'
init_net.Slice(initial_h, initial_h_sliced,
starts=[direction_offset + 0, 0, 0],
ends =[direction_offset + 1,-1,-1])
initial_c_sliced = name + '/initial_c'
init_net.Slice(initial_c, initial_c_sliced,
starts=[direction_offset + 0, 0, 0],
ends =[direction_offset + 1,-1,-1])
if direction_offset == 1:
if sequence_lens is not None:
seq_lens_for_reverse = sequence_lens
else:
input_shape = pred_mh.net.Shape(input_blob, name + '/input_shape')
batch_size = pred_mh.net.Slice(input_shape, name + '/batch_size_slice', starts=[1], ends=[2])
seq_len = pred_mh.net.Slice(input_shape, name + '/seq_len_slice', starts=[0], ends=[1])
dummy_sequence_lens = pred_mh.net.Tile([seq_len, batch_size], name + '/dummy_sequence_lens', axis=0)
pred_mh.net.Reshape(dummy_sequence_lens, [dummy_sequence_lens, cls.dummy_name()], shape=[-1])
seq_lens_for_reverse = dummy_sequence_lens
if direction_offset == 1:
input = pred_mh.net.ReversePackedSegs(
[input_blob, seq_lens_for_reverse], name + "/input-reversed")
else:
input = input_blob
hidden_t_all, hidden_t_last, _, cell_last, params = rnn_cell.LSTM(
pred_mh,
input,
sequence_lens,
[initial_h_sliced, initial_c_sliced],
input_size,
hidden_size,
name,
drop_states=False,
forward_only=True,
return_params=True
)
if direction_offset == 1:
hidden_t_all = pred_mh.net.ReversePackedSegs(
[hidden_t_all, seq_lens_for_reverse], name + "/output-reversed")
return hidden_t_all, hidden_t_last, cell_last
if direction == 'forward':
hidden_t_all, hidden_t_last, cell_last = make_lstm(0)
# in the forward case, storage is shared between the three
# outputs. We need to decouple them so that the
# VariableLengthSequencePadding only mutates n.outputs[0]
pred_mh.net.Copy(hidden_t_last, n.outputs[1])
pred_mh.net.Copy(cell_last, n.outputs[2])
pred_mh.net = pred_mh.net.Clone(
"dummy-clone-net",
blob_remap={ hidden_t_all: n.outputs[0] }
)
elif direction == 'bidirectional':
hidden_t_all_f, hidden_t_last_f, cell_last_f = make_lstm(0)
hidden_t_all_b, hidden_t_last_b, cell_last_b = make_lstm(1)
pred_mh.net.Concat([hidden_t_all_f, hidden_t_all_b],
[n.outputs[0], cls.dummy_name()], axis=2)
pred_mh.net.Concat([hidden_t_last_f, hidden_t_last_b],
[n.outputs[1], cls.dummy_name()], axis=0)
pred_mh.net.Concat([cell_last_f, cell_last_b],
[n.outputs[2], cls.dummy_name()], axis=0)
if sequence_lens is not None:
pred_mh.net.VariableLengthSequencePadding(
[n.outputs[0], sequence_lens], [n.outputs[0]])
return Caffe2Ops(list(pred_mh.Proto().op),
list(init_net.Proto().op),
list(pred_mh.Proto().external_input))
def VGGA(order, cudnn_ws, ideep):
my_arg_scope = {
'order': order,
'use_cudnn': True,
'cudnn_exhaustive_search': True,
'ws_nbytes_limit': str(cudnn_ws)
}
model = ModelHelper(name='vgg-a', arg_scope=my_arg_scope)
conv1 = brew.conv(model,
"data",
"conv1",
3,
64,
3, ('XavierFill', {}), ('ConstantFill', {}),
pad=1)
relu1 = brew.relu(model, conv1, "conv1")
pool1 = brew.max_pool(model, relu1, "pool1", kernel=2, stride=2)
conv2 = brew.conv(model,
pool1,
"conv2",
64,
128,
3, ('XavierFill', {}), ('ConstantFill', {}),
pad=1)
relu2 = brew.relu(model, conv2, "conv2")
pool2 = brew.max_pool(model, relu2, "pool2", kernel=2, stride=2)
conv3 = brew.conv(model,
pool2,
"conv3",
128,
256,
3, ('XavierFill', {}), ('ConstantFill', {}),
pad=1)
relu3 = brew.relu(model, conv3, "conv3")
conv4 = brew.conv(model,
relu3,
"conv4",
256,
256,
3, ('XavierFill', {}), ('ConstantFill', {}),
pad=1)
relu4 = brew.relu(model, conv4, "conv4")
pool4 = brew.max_pool(model, relu4, "pool4", kernel=2, stride=2)
conv5 = brew.conv(model,
pool4,
"conv5",
256,
512,
3, ('XavierFill', {}), ('ConstantFill', {}),
pad=1)
relu5 = brew.relu(model, conv5, "conv5")
conv6 = brew.conv(model,
relu5,
"conv6",
512,
512,
3, ('XavierFill', {}), ('ConstantFill', {}),
pad=1)
relu6 = brew.relu(model, conv6, "conv6")
pool6 = brew.max_pool(model, relu6, "pool6", kernel=2, stride=2)
conv7 = brew.conv(model,
pool6,
"conv7",
512,
512,
3, ('XavierFill', {}), ('ConstantFill', {}),
pad=1)
relu7 = brew.relu(model, conv7, "conv7")
conv8 = brew.conv(model,
relu7,
"conv8",
512,
512,
3, ('XavierFill', {}), ('ConstantFill', {}),
pad=1)
relu8 = brew.relu(model, conv8, "conv8")
pool8 = brew.max_pool(model, relu8, "pool8", kernel=2, stride=2)
fcix = brew.fc(model, pool8, "fcix", 512 * 7 * 7, 4096, ('XavierFill', {}),
('ConstantFill', {}))
reluix = brew.relu(model, fcix, "fcix")
fcx = brew.fc(model, reluix, "fcx", 4096, 4096, ('XavierFill', {}),
('ConstantFill', {}))
relux = brew.relu(model, fcx, "fcx")
fcxi = brew.fc(model, relux, "fcxi", 4096, 1000, ('XavierFill', {}),
('ConstantFill', {}))
pred = brew.softmax(model, fcxi, "pred")
xent = model.LabelCrossEntropy([pred, "label"], "xent")
loss = model.AveragedLoss(xent, "loss")
return model, 231
plt.scatter([i[0] for i in X], Y_gt, label="original data", color='b')
plt.xlabel("x")
plt.ylabel("y")
plt.title("Input Training Data")
# ### Create the Model
#
# #### Define the model architecture
# With our training data created and our second order polynomial assumption stated, we can now create a model to learn the regression line. We will use a 'FC' layer as the main component of the model. Since we desire two weights ($\beta_2$ and $\beta_1$), we set our input dimension to 2, and since we only expect a single quantitative result, our output dimension is 1. Note, when using an 'FC' layer it is implied that there is a bias, which we will use as our $\beta_0.$
#
# Also, before continuing take a look at the protobuf created in this step. The first print out is of the 'net,' and contains the architecture of the model. At a glance, we see that as expected, there is a single op in the network that expects an input $X,$ a weight and bias, and outputs $y_{pred}.$ In the print out of the 'param_init_net,' we see that this is where the initializations for the weights and biases exist. This is an important observation that gives insight into how a model in Caffe2 is constructed and maintained.
# In[4]:
# Create the model helper object we will use to create the regression model
regression_model = ModelHelper(name="regression_model")
# Add the FC layer, which is the main component of a linear regression model
y_pred = brew.fc(regression_model, 'X', 'y_pred', dim_in=2, dim_out=1)
# Print the predict and init net to see what protobuf was created for this model
print("************* Predict Net *************")
print(regression_model.net.Proto())
print("\n************* Init Net *************")
print(regression_model.param_init_net.Proto())
# #### Add the training operators and prime the workspace
#
# In this **very important** step, we specify the loss function, setup the SGD training algorithm, prime and initialize the workspace, and initialize our model's weights and biases.
# In[5]:
