def test_compute_and_gradients():
"""Tests the Network's compute and compute_gradients methods.
"""
batch = np.random.randint(1, 128)
input_dims = np.random.randint(1, 256)
output_dims = np.random.randint(1, 512)
inputs = np.random.uniform(size=(batch, input_dims))
weights = np.random.uniform(size=(input_dims, output_dims))
biases = np.random.uniform(size=(output_dims))
fullyconnected_layer = FullyConnectedLayer(weights, biases)
relu_layer = ReluLayer()
fullyconnected_outputs = fullyconnected_layer.compute(inputs)
relu_outputs = relu_layer.compute(fullyconnected_outputs)
network = Network([fullyconnected_layer, relu_layer])
network_outputs = network.compute(inputs)
assert np.allclose(network_outputs, relu_outputs)
assert np.allclose(network_outputs, network.compute(list(inputs)))
assert np.allclose(network_outputs[0], network.compute(list(inputs)[0]))
for label in range(output_dims):
gradients = network.compute_gradients(inputs, label)
for i in range(batch):
if fullyconnected_outputs[i, label] <= 0.0:
assert np.allclose(gradients[i], 0.0)
else:
assert np.allclose(gradients[i], weights[:, label])
def test_compute_invalid():
"""Tests that the Concat layer fails on an invalid concat_along.
"""
batch = 15
n_inputs = 1025
fullyconnected_outputs = 2046
inputs = np.random.uniform(size=(batch, n_inputs)).astype(np.float32)
weights = np.random.uniform(size=(n_inputs, fullyconnected_outputs))
weights = weights.astype(np.float32)
biases = np.random.uniform(size=(fullyconnected_outputs))
biases = biases.astype(np.float32)
fullyconnected_layer = FullyConnectedLayer(weights, biases)
means = np.random.uniform(size=(n_inputs)).astype(np.float32)
stds = np.random.uniform(size=(n_inputs)).astype(np.float32)
normalize_layer = NormalizeLayer(means, stds)
concat_layer = ConcatLayer([fullyconnected_layer, normalize_layer], None)
try:
concat_layer.compute(inputs)
assert False
except NotImplementedError:
assert True
def test_compute_flat():
"""Tests that the Concat layer correctly computes.
Uses concat_along = FLAT
"""
batch = 15
n_inputs = 1025
fullyconnected_outputs = 2046
inputs = np.random.uniform(size=(batch, n_inputs)).astype(np.float32)
weights = np.random.uniform(size=(n_inputs, fullyconnected_outputs))
weights = weights.astype(np.float32)
biases = np.random.uniform(size=(fullyconnected_outputs))
biases = biases.astype(np.float32)
true_fullyconnected_outputs = np.matmul(inputs, weights) + biases
means = np.random.uniform(size=(n_inputs)).astype(np.float32)
stds = np.random.uniform(size=(n_inputs)).astype(np.float32)
true_normalize_outputs = (inputs - means) / stds
true_outputs = np.concatenate([true_fullyconnected_outputs,
true_normalize_outputs],
axis=1)
fullyconnected_layer = FullyConnectedLayer(weights, biases)
normalize_layer = NormalizeLayer(means, stds)
concat_layer = ConcatLayer([fullyconnected_layer, normalize_layer],
ConcatAlong.FLAT)
assert np.allclose(concat_layer.compute(inputs), true_outputs)
torch_inputs = torch.FloatTensor(inputs)
torch_outputs = concat_layer.compute(torch_inputs).numpy()
assert np.allclose(torch_outputs, true_outputs)
def test_serialize():
"""Tests that the Concat layer correctly serializes itself.
"""
n_inputs = 125
fullyconnected_outputs = 246
weights = np.random.uniform(size=(n_inputs, fullyconnected_outputs))
biases = np.random.uniform(size=(fullyconnected_outputs))
fullyconnected_layer = FullyConnectedLayer(weights, biases)
means = np.random.uniform(size=(n_inputs)).astype(np.float32)
stds = np.random.uniform(size=(n_inputs)).astype(np.float32)
normalize_layer = NormalizeLayer(means, stds)
concat_layer = ConcatLayer([fullyconnected_layer, normalize_layer],
ConcatAlong.FLAT)
serialized = concat_layer.serialize()
assert serialized.WhichOneof("layer_data") == "concat_data"
assert (serialized.concat_data.concat_along ==
transformer_pb.ConcatLayerData.ConcatAlong.Value(
"CONCAT_ALONG_FLAT"))
assert len(serialized.concat_data.layers) == 2
assert serialized.concat_data.layers[0] == fullyconnected_layer.serialize()
assert serialized.concat_data.layers[1] == normalize_layer.serialize()
# TODO: This does not check that deserialized.input_layers was done
# correctly, but that should be the case as long as their deserialize
# methods work (tested in their respective files).
deserialized = ConcatLayer.deserialize(serialized)
assert deserialized.concat_along == ConcatAlong.FLAT
serialized.relu_data.SetInParent()
deserialized = ConcatLayer.deserialize(serialized)
assert deserialized is None
concat_layer.concat_along = ConcatAlong.CHANNELS
deserialized = ConcatLayer.deserialize(concat_layer.serialize())
assert deserialized.concat_along == ConcatAlong.CHANNELS
try:
ConcatAlong.deserialize(5)
assert False, "Should have errored on unrecognized serialization."
except NotImplementedError:
pass
def test_compute():
"""Tests that the Fully-Connected layer correctly computes.
"""
n_inputs = 1025
n_outputs = 2046
batch = 15
inputs = np.random.uniform(size=(batch, n_inputs))
weights = np.random.uniform(size=(n_inputs, n_outputs))
biases = np.random.uniform(size=(n_outputs))
true_outputs = np.matmul(inputs, weights) + biases
fullyconnected_layer = FullyConnectedLayer(weights, biases)
assert np.allclose(fullyconnected_layer.compute(inputs), true_outputs)
torch_inputs = torch.FloatTensor(inputs)
torch_outputs = fullyconnected_layer.compute(torch_inputs).numpy()
assert np.allclose(torch_outputs, true_outputs)
def test_serialize():
"""Tests the Network's serialize and deserialize methods.
"""
input_dims = np.random.randint(1, 32)
output_dims = np.random.randint(1, 64)
weights = np.random.uniform(size=(input_dims, output_dims))
biases = np.random.uniform(size=(output_dims))
fullyconnected_layer = FullyConnectedLayer(weights, biases)
relu_layer = ReluLayer()
network = Network([fullyconnected_layer, relu_layer])
serialized = network.serialize()
assert len(serialized.layers) == 2
assert serialized.layers[0] == fullyconnected_layer.serialize()
assert serialized.layers[1] == relu_layer.serialize()
deserialized = Network.deserialize(serialized)
assert deserialized.serialize() == serialized
def test_serialize():
"""Tests that the Concat layer correctly serializes itself.
"""
n_inputs = 1025
fullyconnected_outputs = 2046
weights = np.random.uniform(size=(n_inputs, fullyconnected_outputs))
biases = np.random.uniform(size=(fullyconnected_outputs))
fullyconnected_layer = FullyConnectedLayer(weights, biases)
means = np.random.uniform(size=(n_inputs)).astype(np.float32)
stds = np.random.uniform(size=(n_inputs)).astype(np.float32)
normalize_layer = NormalizeLayer(means, stds)
concat_layer = ConcatLayer([fullyconnected_layer, normalize_layer],
ConcatAlong.FLAT)
serialized = concat_layer.serialize()
assert serialized.WhichOneof("layer_data") == "concat_data"
assert (serialized.concat_data.concat_along == transformer_pb.
ConcatLayerData.ConcatAlong.Value("CONCAT_ALONG_FLAT"))
assert len(serialized.concat_data.layers) == 2
assert serialized.concat_data.layers[0] == fullyconnected_layer.serialize()
assert serialized.concat_data.layers[1] == normalize_layer.serialize()
def test_serialize():
"""Tests that the Fully-Connected layer correctly [de]serializes itself.
"""
n_inputs = 129
n_outputs = 291
weights = np.random.uniform(size=(n_inputs, n_outputs))
biases = np.random.uniform(size=(n_outputs))
serialized = FullyConnectedLayer(weights, biases).serialize()
assert serialized.WhichOneof("layer_data") == "fullyconnected_data"
serialized_weights = np.array(serialized.fullyconnected_data.weights)
assert np.allclose(serialized_weights.flatten(), weights.flatten())
serialized_biases = np.array(serialized.fullyconnected_data.biases)
assert np.allclose(serialized_biases.flatten(), biases.flatten())
deserialized = FullyConnectedLayer.deserialize(serialized)
assert deserialized.serialize() == serialized
serialized.relu_data.SetInParent()
deserialized = FullyConnectedLayer.deserialize(serialized)
assert FullyConnectedLayer.deserialize(serialized) is None
def test_exactlines():
import pysyrenn.frontend.transformer_client
transform_lines_ = pysyrenn.frontend.transformer_client.transform_lines
input_dims = np.random.randint(1, 32)
output_dims = np.random.randint(1, 64)
weights = np.random.uniform(size=(input_dims, output_dims))
biases = np.random.uniform(size=(output_dims))
fullyconnected_layer = FullyConnectedLayer(weights, biases)
relu_layer = ReluLayer()
network = Network([fullyconnected_layer, relu_layer])
lines = list(np.random.uniform(size=(100, 2, input_dims)))
def transform_lines_mock(query_network,
query_lines,
query_include_post=False):
assert query_network.serialize() == network.serialize()
if len(query_lines) == 1:
assert np.allclose(query_lines, lines[:1])
else:
assert np.allclose(query_lines, lines)
output_lines = []
for i, line in enumerate(query_lines):
output_lines.append((np.array([0.0, 1.0 / float(i + 1),
1.0]), np.array([float(2.0 * i)])))
return output_lines
pysyrenn.frontend.transformer_client.transform_lines = transform_lines_mock
ratios = network.exactlines(lines,
compute_preimages=False,
include_post=False)
assert np.allclose(
ratios, np.array([[0.0, 1.0 / float(i + 1), 1.0] for i in range(100)]))
ratio = network.exactline(*lines[0],
compute_preimages=False,
include_post=False)
assert np.allclose(ratio, ratios[0])
def interpolate(line_i, ratio):
start, end = lines[line_i]
return start + (ratio * (end - start))
preimages = network.exactlines(lines,
compute_preimages=True,
include_post=False)
assert np.allclose(
preimages,
np.array([[
interpolate(i, 0.0),
interpolate(i, 1.0 / float(i + 1)),
interpolate(i, 1.0)
] for i in range(100)]))
preimage = network.exactline(*lines[0],
compute_preimages=True,
include_post=False)
assert np.allclose(preimage, preimages[0])
transformed = network.exactlines(lines,
compute_preimages=True,
include_post=True)
pre, post = zip(*transformed)
assert np.allclose(
pre,
np.array([[
interpolate(i, 0.0),
interpolate(i, 1.0 / float(i + 1)),
interpolate(i, 1.0)
] for i in range(100)]))
assert np.allclose(post, np.array([[float(2.0 * i)] for i in range(100)]))
transformed_single = network.exactline(*lines[0],
compute_preimages=True,
include_post=True)
assert np.allclose(transformed_single[0], transformed[0][0])
assert np.allclose(transformed_single[1], transformed[0][1])
def test_transform_lines():
open_stub_ = transformer_client.open_stub
input_dims = np.random.randint(1, 32)
output_dims = np.random.randint(1, 64)
weights = np.random.uniform(size=(input_dims, output_dims))
biases = np.random.uniform(size=(output_dims))
fullyconnected_layer = FullyConnectedLayer(weights, biases)
relu_layer = ReluLayer()
network = Network([fullyconnected_layer, relu_layer])
lines = list(np.random.uniform(size=(100, 2, input_dims)))
response_messages = []
for i, line in enumerate(lines):
transformed_line = transformer_pb.SegmentedLine()
for j in range(i + 2):
endpoint = transformer_pb.SegmentEndpoint()
endpoint.coordinates.extend(np.arange(i, i + 10))
endpoint.preimage_ratio = j / ((i + 2) - 1)
transformed_line.endpoints.append(endpoint)
response = transformer_pb.TransformResponse()
response.transformed_line.CopyFrom(transformed_line)
response_messages.append(response)
# With include_post = True.
stub = ServerStubMock(response_messages)
transformer_client.open_stub = lambda: stub
transformed_lines = transformer_client.transform_lines(network,
lines,
include_post=True)
def verify_response(stub, transformed_lines, included_post):
assert len(transformed_lines) == len(lines)
for i, line in enumerate(lines):
transformed_pre, transformed_post = transformed_lines[i]
assert len(transformed_pre) == (i + 2)
assert np.allclose(transformed_pre,
[j / ((i + 2) - 1) for j in range(i + 2)])
if included_post:
assert len(transformed_post) == (i + 2)
assert np.allclose(transformed_post, np.arange(i, i + 10))
else:
assert transformed_post is None
assert len(stub.received_messages) == 1
received = stub.received_messages[0]
assert len(received) == 102
assert received[0].WhichOneof("request_data") == "layer"
assert received[0].layer == fullyconnected_layer.serialize()
assert received[1].layer == relu_layer.serialize()
for i, request in enumerate(received[2:]):
assert request.WhichOneof("request_data") == "line"
assert len(request.line.endpoints) == 2
assert request.line.endpoints[0].preimage_ratio == 0.0
assert request.line.endpoints[1].preimage_ratio == 1.0
assert np.allclose(np.array(request.line.endpoints[0].coordinates),
lines[i][0])
assert np.allclose(np.array(request.line.endpoints[1].coordinates),
lines[i][1])
verify_response(stub, transformed_lines, True)
# With include_post = False.
for response_message in response_messages:
for endpoint in response_message.transformed_line.endpoints:
while endpoint.coordinates:
endpoint.coordinates.pop()
stub = ServerStubMock(response_messages)
transformer_client.open_stub = lambda: stub
transformed_lines = transformer_client.transform_lines(network,
lines,
include_post=False)
verify_response(stub, transformed_lines, False)
transformer_client.open_stub = open_stub_
def test_transform_planes():
open_stub_ = transformer_client.open_stub
input_dims = np.random.randint(1, 32)
output_dims = np.random.randint(1, 64)
weights = np.random.uniform(size=(input_dims, output_dims))
biases = np.random.uniform(size=(output_dims))
fullyconnected_layer = FullyConnectedLayer(weights, biases)
relu_layer = ReluLayer()
network = Network([fullyconnected_layer, relu_layer])
planes = list(np.random.uniform(size=(100, 3, input_dims)))
response_messages = []
for i, plane in enumerate(planes):
transformed_polytope = transformer_pb.UPolytope()
transformed_polytope.space_dimensions = output_dims
transformed_polytope.subspace_dimensions = 2
for j in range(i + 2):
polytope = transformer_pb.VPolytope()
polytope.vertices.extend(np.matmul(plane, weights).flatten())
polytope.combinations.extend(np.eye(3).flatten())
polytope.num_vertices = 3
transformed_polytope.polytopes.append(polytope)
response = transformer_pb.TransformResponse()
response.transformed_upolytope.CopyFrom(transformed_polytope)
response_messages.append(response)
# With include_post = True.
stub = ServerStubMock(response_messages)
transformer_client.open_stub = lambda: stub
transformed = transformer_client.transform_planes(network, planes)
assert len(transformed) == len(planes)
for i, plane in enumerate(planes):
upolytope = transformed[i]
assert len(upolytope) == (i + 2)
for vpolytope in upolytope:
transformed_pre, transformed_post = vpolytope
assert len(transformed_pre) == len(transformed_post) == 3
assert np.allclose(transformed_pre, np.eye(3))
assert np.allclose(transformed_post, np.matmul(plane, weights))
assert len(stub.received_messages) == 1
received = stub.received_messages[0]
assert len(received) == 102
assert received[0].WhichOneof("request_data") == "layer"
assert received[0].layer == fullyconnected_layer.serialize()
assert received[1].layer == relu_layer.serialize()
for i, request in enumerate(received[2:]):
assert request.WhichOneof("request_data") == "upolytope"
assert request.upolytope.space_dimensions == input_dims
assert request.upolytope.subspace_dimensions == 2
assert len(request.upolytope.polytopes) == 1
assert request.upolytope.polytopes[0].num_vertices == 3
assert np.allclose(request.upolytope.polytopes[0].vertices,
planes[i].flatten())
assert np.allclose(request.upolytope.polytopes[0].combinations,
np.eye(3).flatten())
def layer_from_onnx(graph, node):
"""Reads a layer from an ONNX node.
Specs for the ONNX operators are available at:
https://github.com/onnx/onnx/blob/master/docs/Operators.md
"""
# First, we get info about inputs to the layer (including previous
# layer outputs & things like weight matrices).
inputs = node.input
deserialized_inputs = []
deserialized_input_shapes = []
for input_name in inputs:
# We need to find the initializers (which I think are basically
# weight tensors) for the particular input.
initializers = [init for init in graph.initializer
if str(init.name) == str(input_name)]
if initializers:
assert len(initializers) == 1
# Get the weight tensor as a Numpy array and save it.
deserialized_inputs.append(numpy_helper.to_array(initializers[0]))
else:
# This input is the output of another node, so just store the
# name of that other node (we'll link them up later). Eg.
# squeezenet0_conv0_fwd.
deserialized_inputs.append(str(input_name))
# Get metadata about the input (eg. its shape).
infos = [info for info in graph.value_info
if info.name == input_name]
if infos:
# This is an input with a particular shape.
assert len(infos) == 1
input_shape = [d.dim_value
for d in infos[0].type.tensor_type.shape.dim]
deserialized_input_shapes.append(input_shape)
elif input_name == "data":
# This is an input to the entire network, its handled
# separately.
net_input_shape = graph.input[0].type.tensor_type.shape
input_shape = [d.dim_value for d in net_input_shape.dim]
deserialized_input_shapes.append(input_shape)
else:
# This doesn't have any inputs.
deserialized_input_shapes.append(None)
layer = None
# Standardize some of the data shared by the strided-window layers.
if node.op_type in {"Conv", "MaxPool", "AveragePool"}:
# NCHW -> NHWC
input_shape = deserialized_input_shapes[0]
input_shape = [input_shape[2], input_shape[3], input_shape[1]]
strides = list(Network.onnx_ints_attribute(node, "strides"))
pads = list(Network.onnx_ints_attribute(node, "pads"))
# We do not support separate begin/end padding.
assert pads[0] == pads[2]
assert pads[1] == pads[3]
pads = pads[1:3]
# Now, parse the actual layers.
if node.op_type == "Conv":
# We don't support dilations or non-1 groups.
dilations = list(Network.onnx_ints_attribute(node, "dilations"))
assert all(dilation == 1 for dilation in dilations)
group = Network.onnx_ints_attribute(node, "group")
assert not group or group == 1
# biases are technically optional, but I don't *think* anyone uses
# that feature.
assert len(deserialized_inputs) == 3
input_data, filters, biases = deserialized_inputs
# OIHW -> HWIO
filters = filters.transpose((2, 3, 1, 0))
window_data = StridedWindowData(input_shape, filters.shape[:2],
strides, pads, biases.shape[0])
layer = Conv2DLayer(window_data, filters, biases)
elif node.op_type == "Relu":
layer = ReluLayer()
elif node.op_type == "MaxPool":
kernel_shape = Network.onnx_ints_attribute(node, "kernel_shape")
window_data = StridedWindowData(input_shape, list(kernel_shape),
strides, pads, input_shape[2])
layer = MaxPoolLayer(window_data)
elif node.op_type == "AveragePool":
kernel_shape = Network.onnx_ints_attribute(node, "kernel_shape")
window_data = StridedWindowData(input_shape, list(kernel_shape),
strides, pads, input_shape[2])
layer = AveragePoolLayer(window_data)
elif node.op_type == "Gemm":
input_data, weights, biases = deserialized_inputs
alpha = Network.onnx_ints_attribute(node, "alpha")
if alpha:
weights *= alpha
beta = Network.onnx_ints_attribute(node, "beta")
if beta:
biases *= beta
trans_A = Network.onnx_ints_attribute(node, "transA")
trans_B = Network.onnx_ints_attribute(node, "transB")
# We compute (X . W) [+ C].
assert not trans_A
if trans_B:
weights = weights.transpose()
layer = FullyConnectedLayer(weights, biases)
elif node.op_type == "BatchNormalization":
epsilon = Network.onnx_ints_attribute(node, "epsilon")
input_data, scale, B, mean, var = deserialized_inputs
# We don't yet support separate scale/bias parameters, though they
# can be rolled in to mean/var.
assert np.allclose(scale, 1.0) and np.allclose(B, 0.0)
layer = NormalizeLayer(mean, np.sqrt(var + epsilon))
elif node.op_type == "Concat":
layer = list(inputs)
elif node.op_type in {"Dropout", "Reshape", "Flatten"}:
# These are (more-or-less) handled implicitly since we pass around
# flattened activation vectors and only work with testing.
layer = False
else:
raise NotImplementedError
assert len(node.output) == 1
return (inputs[0], node.output[0], layer)
def from_eran(cls, net_file):
"""Helper method to read an ERAN net_file into a Network.
Currently only supports a subset of those supported by the original
read_net_file.py. See an example of the type of network file we're
reading here:
https://files.sri.inf.ethz.ch/eran/nets/tensorflow/mnist/mnist_relu_3_100.tf
This code has been adapted (with heavy modifications) from the ERAN
source code. Each layer has a header line that describes the type of
layer, which is then followed by the weights (if applicable). Note that
some layers are rolled together in ERAN but we do separately (eg.
"ReLU" in the ERAN format corresponds to Affine + ReLU in our
representation).
"""
layers = []
net_file = open(net_file, "r")
while True:
curr_line = net_file.readline()[:-1]
if curr_line in {"Affine", "ReLU", "HardTanh"}:
# Parses a fully-connected layer, possibly followed by
# non-linearity.
# ERAN files use (out_dims, in_dims), we use the opposite.
weights = cls.parse_np_array(net_file).transpose()
biases = cls.parse_np_array(net_file)
if len(layers) > 1 and isinstance(layers[-2], Conv2DLayer):
# When there's an affine after a 2D convolution, ERAN's
# files assume the input is CHW when it's actually HWC. We
# correct that here by permuting the dimensions.
conv_layer = layers[-2]
output_size = weights.shape[-1]
weights = weights.reshape(
(conv_layer.window_data.out_channels,
conv_layer.window_data.out_height(),
conv_layer.window_data.out_width(),
output_size))
weights = weights.transpose(1, 2, 0, 3)
weights = weights.reshape((-1, output_size))
# Add the fully-connected layer.
layers.append(FullyConnectedLayer(weights, biases))
# Maybe add a non-linearity.
if curr_line == "ReLU":
layers.append(ReluLayer())
elif curr_line == "HardTanh":
layers.append(HardTanhLayer())
elif curr_line.startswith("Normalize"):
# Parses a Normalize layer.
means = curr_line.split("mean=")[1].split("std=")[0].strip()
means = cls.parse_np_array(means)
stds = curr_line.split("std=")[1].strip()
stds = cls.parse_np_array(stds)
layers.append(NormalizeLayer(means, stds))
elif curr_line.startswith("Conv2D"):
# Parses a 2D-Convolution layer. The info line looks like:
# ReLU, filters=16, kernel_size=[4, 4], \
# input_shape=[28, 28, 1], stride=[2, 2], padding=0
# But, we can get filters and kernel_size from the actual
# filter weights, so no need to parse that here.
info_line = net_file.readline()[:-1].strip()
activation = info_line.split(",")[0]
stride = cls.parse_np_array(
info_line.split("stride=")[1].split("],")[0] + "]")
input_shape = info_line.split("input_shape=")[1].split("],")[0]
input_shape += "]"
input_shape = cls.parse_np_array(input_shape)
pad = (0, 0)
if "padding=" in info_line:
pad = int(info_line.split("padding=")[1])
pad = (pad, pad)
# (f_h, f_w, i_c, o_c)
filter_weights = cls.parse_np_array(net_file)
# (o_c,)
biases = cls.parse_np_array(net_file)
window_data = StridedWindowData(
input_shape, filter_weights.shape[:2],
stride, pad, filter_weights.shape[3])
layers.append(Conv2DLayer(window_data, filter_weights, biases))
if activation == "ReLU":
layers.append(ReluLayer())
elif activation == "HardTanh":
layers.append(HardTanhLayer())
else:
# As far as I know, all Conv2D layers should have an
# associated activation function in the ERAN format.
raise NotImplementedError
elif curr_line.strip() == "":
break
else:
raise NotImplementedError
return cls(layers)