def test_compute_and_gradients():
"""Tests the Network's compute and compute_gradients methods.
"""
batch = np.random.randint(1, 128)
input_dims = np.random.randint(1, 256)
output_dims = np.random.randint(1, 512)
inputs = np.random.uniform(size=(batch, input_dims))
weights = np.random.uniform(size=(input_dims, output_dims))
biases = np.random.uniform(size=(output_dims))
fullyconnected_layer = FullyConnectedLayer(weights, biases)
relu_layer = ReluLayer()
fullyconnected_outputs = fullyconnected_layer.compute(inputs)
relu_outputs = relu_layer.compute(fullyconnected_outputs)
network = Network([fullyconnected_layer, relu_layer])
network_outputs = network.compute(inputs)
assert np.allclose(network_outputs, relu_outputs)
assert np.allclose(network_outputs, network.compute(list(inputs)))
assert np.allclose(network_outputs[0], network.compute(list(inputs)[0]))
for label in range(output_dims):
gradients = network.compute_gradients(inputs, label)
for i in range(batch):
if fullyconnected_outputs[i, label] <= 0.0:
assert np.allclose(gradients[i], 0.0)
else:
assert np.allclose(gradients[i], weights[:, label])
def test_serialize():
"""Tests that the ReLU layer correctly [de]serializes itself.
"""
serialized = ReluLayer().serialize()
assert serialized.WhichOneof("layer_data") == "relu_data"
deserialized = ReluLayer.deserialize(serialized)
assert deserialized.serialize() == serialized
serialized.normalize_data.SetInParent()
assert ReluLayer.deserialize(serialized) is None
def test_compute():
"""Tests that the ReLU layer correctly computes a ReLU.
"""
inputs = np.random.uniform(size=(101, 1025))
true_relu = np.maximum(inputs, 0.0)
relu_layer = ReluLayer()
assert np.allclose(relu_layer.compute(inputs), true_relu)
torch_inputs = torch.FloatTensor(inputs)
torch_outputs = relu_layer.compute(torch_inputs).numpy()
assert np.allclose(torch_outputs, true_relu)
def test_serialize():
"""Tests the Network's serialize and deserialize methods.
"""
input_dims = np.random.randint(1, 32)
output_dims = np.random.randint(1, 64)
weights = np.random.uniform(size=(input_dims, output_dims))
biases = np.random.uniform(size=(output_dims))
fullyconnected_layer = FullyConnectedLayer(weights, biases)
relu_layer = ReluLayer()
network = Network([fullyconnected_layer, relu_layer])
serialized = network.serialize()
assert len(serialized.layers) == 2
assert serialized.layers[0] == fullyconnected_layer.serialize()
assert serialized.layers[1] == relu_layer.serialize()
deserialized = Network.deserialize(serialized)
assert deserialized.serialize() == serialized
def has_connection(cls):
"""Returns True iff the transformer server can be reached.
"""
try:
network = cls([ReluLayer()])
network.exactline([0.0], [1.0], False, False)
return True
except Exception as exception:
if "failed to connect" in exception.details():
return False
raise exception
def from_onnx(cls, net_file):
"""Reads a network from an ONNX file.
"""
model = onnx.load(net_file)
model = shape_inference.infer_shapes(model)
# layers will be {output_name: layer}
layers = {}
# First, we just convert everything we can into a layer
for node in model.graph.node:
layer = cls.layer_from_onnx(model.graph, node)
if layer is not False:
input_name, output_name, layer = layer
layers[output_name] = (input_name, layer)
# Then, we roll all of the concat layers together
while any(l for l in layers.values() if isinstance(l[1], list)):
concat_name = next(name for name, layer in layers.items()
if isinstance(layer[1], list))
_, input_layers = layers[concat_name]
assert all(isinstance(layers[input_name][1], ReluLayer)
for input_name in input_layers)
# We move the relus to behind the concat
relu_layer_names = input_layers.copy()
input_layer_names = [layers[input_name][0]
for input_name in relu_layer_names]
entire_input_name = layers[input_layer_names[0]][0]
assert all(entire_input_name == layers[input_layer][0]
for input_layer in input_layer_names)
input_layers = [layers[input_name][1]
for input_name in input_layer_names]
for input_layer_name in input_layer_names:
layers.pop(input_layer_name)
# Then we remove all of the intermediate relu layers
for relu_layer_name in relu_layer_names:
layers.pop(relu_layer_name)
concat_layer = ConcatLayer(input_layers)
layers[concat_name + "_prerelu"] = (entire_input_name, concat_layer)
layers[concat_name] = (concat_name + "_prerelu", ReluLayer())
# Then, we flatten
flat_layers = []
input_name = "data"
while layers:
next_input_name, next_layer = next(
(output_name, layer[1])
for output_name, layer in layers.items()
if layer[0] == input_name)
flat_layers.append(next_layer)
input_name = next_input_name
layers.pop(next_input_name)
flat_layers = [layer for layer in flat_layers if layer is not False]
return cls(flat_layers)
def test_exactlines():
import pysyrenn.frontend.transformer_client
transform_lines_ = pysyrenn.frontend.transformer_client.transform_lines
input_dims = np.random.randint(1, 32)
output_dims = np.random.randint(1, 64)
weights = np.random.uniform(size=(input_dims, output_dims))
biases = np.random.uniform(size=(output_dims))
fullyconnected_layer = FullyConnectedLayer(weights, biases)
relu_layer = ReluLayer()
network = Network([fullyconnected_layer, relu_layer])
lines = list(np.random.uniform(size=(100, 2, input_dims)))
def transform_lines_mock(query_network,
query_lines,
query_include_post=False):
assert query_network.serialize() == network.serialize()
if len(query_lines) == 1:
assert np.allclose(query_lines, lines[:1])
else:
assert np.allclose(query_lines, lines)
output_lines = []
for i, line in enumerate(query_lines):
output_lines.append((np.array([0.0, 1.0 / float(i + 1),
1.0]), np.array([float(2.0 * i)])))
return output_lines
pysyrenn.frontend.transformer_client.transform_lines = transform_lines_mock
ratios = network.exactlines(lines,
compute_preimages=False,
include_post=False)
assert np.allclose(
ratios, np.array([[0.0, 1.0 / float(i + 1), 1.0] for i in range(100)]))
ratio = network.exactline(*lines[0],
compute_preimages=False,
include_post=False)
assert np.allclose(ratio, ratios[0])
def interpolate(line_i, ratio):
start, end = lines[line_i]
return start + (ratio * (end - start))
preimages = network.exactlines(lines,
compute_preimages=True,
include_post=False)
assert np.allclose(
preimages,
np.array([[
interpolate(i, 0.0),
interpolate(i, 1.0 / float(i + 1)),
interpolate(i, 1.0)
] for i in range(100)]))
preimage = network.exactline(*lines[0],
compute_preimages=True,
include_post=False)
assert np.allclose(preimage, preimages[0])
transformed = network.exactlines(lines,
compute_preimages=True,
include_post=True)
pre, post = zip(*transformed)
assert np.allclose(
pre,
np.array([[
interpolate(i, 0.0),
interpolate(i, 1.0 / float(i + 1)),
interpolate(i, 1.0)
] for i in range(100)]))
assert np.allclose(post, np.array([[float(2.0 * i)] for i in range(100)]))
transformed_single = network.exactline(*lines[0],
compute_preimages=True,
include_post=True)
assert np.allclose(transformed_single[0], transformed[0][0])
assert np.allclose(transformed_single[1], transformed[0][1])
def test_transform_lines():
open_stub_ = transformer_client.open_stub
input_dims = np.random.randint(1, 32)
output_dims = np.random.randint(1, 64)
weights = np.random.uniform(size=(input_dims, output_dims))
biases = np.random.uniform(size=(output_dims))
fullyconnected_layer = FullyConnectedLayer(weights, biases)
relu_layer = ReluLayer()
network = Network([fullyconnected_layer, relu_layer])
lines = list(np.random.uniform(size=(100, 2, input_dims)))
response_messages = []
for i, line in enumerate(lines):
transformed_line = transformer_pb.SegmentedLine()
for j in range(i + 2):
endpoint = transformer_pb.SegmentEndpoint()
endpoint.coordinates.extend(np.arange(i, i + 10))
endpoint.preimage_ratio = j / ((i + 2) - 1)
transformed_line.endpoints.append(endpoint)
response = transformer_pb.TransformResponse()
response.transformed_line.CopyFrom(transformed_line)
response_messages.append(response)
# With include_post = True.
stub = ServerStubMock(response_messages)
transformer_client.open_stub = lambda: stub
transformed_lines = transformer_client.transform_lines(network,
lines,
include_post=True)
def verify_response(stub, transformed_lines, included_post):
assert len(transformed_lines) == len(lines)
for i, line in enumerate(lines):
transformed_pre, transformed_post = transformed_lines[i]
assert len(transformed_pre) == (i + 2)
assert np.allclose(transformed_pre,
[j / ((i + 2) - 1) for j in range(i + 2)])
if included_post:
assert len(transformed_post) == (i + 2)
assert np.allclose(transformed_post, np.arange(i, i + 10))
else:
assert transformed_post is None
assert len(stub.received_messages) == 1
received = stub.received_messages[0]
assert len(received) == 102
assert received[0].WhichOneof("request_data") == "layer"
assert received[0].layer == fullyconnected_layer.serialize()
assert received[1].layer == relu_layer.serialize()
for i, request in enumerate(received[2:]):
assert request.WhichOneof("request_data") == "line"
assert len(request.line.endpoints) == 2
assert request.line.endpoints[0].preimage_ratio == 0.0
assert request.line.endpoints[1].preimage_ratio == 1.0
assert np.allclose(np.array(request.line.endpoints[0].coordinates),
lines[i][0])
assert np.allclose(np.array(request.line.endpoints[1].coordinates),
lines[i][1])
verify_response(stub, transformed_lines, True)
# With include_post = False.
for response_message in response_messages:
for endpoint in response_message.transformed_line.endpoints:
while endpoint.coordinates:
endpoint.coordinates.pop()
stub = ServerStubMock(response_messages)
transformer_client.open_stub = lambda: stub
transformed_lines = transformer_client.transform_lines(network,
lines,
include_post=False)
verify_response(stub, transformed_lines, False)
transformer_client.open_stub = open_stub_
def test_transform_planes():
open_stub_ = transformer_client.open_stub
input_dims = np.random.randint(1, 32)
output_dims = np.random.randint(1, 64)
weights = np.random.uniform(size=(input_dims, output_dims))
biases = np.random.uniform(size=(output_dims))
fullyconnected_layer = FullyConnectedLayer(weights, biases)
relu_layer = ReluLayer()
network = Network([fullyconnected_layer, relu_layer])
planes = list(np.random.uniform(size=(100, 3, input_dims)))
response_messages = []
for i, plane in enumerate(planes):
transformed_polytope = transformer_pb.UPolytope()
transformed_polytope.space_dimensions = output_dims
transformed_polytope.subspace_dimensions = 2
for j in range(i + 2):
polytope = transformer_pb.VPolytope()
polytope.vertices.extend(np.matmul(plane, weights).flatten())
polytope.combinations.extend(np.eye(3).flatten())
polytope.num_vertices = 3
transformed_polytope.polytopes.append(polytope)
response = transformer_pb.TransformResponse()
response.transformed_upolytope.CopyFrom(transformed_polytope)
response_messages.append(response)
# With include_post = True.
stub = ServerStubMock(response_messages)
transformer_client.open_stub = lambda: stub
transformed = transformer_client.transform_planes(network, planes)
assert len(transformed) == len(planes)
for i, plane in enumerate(planes):
upolytope = transformed[i]
assert len(upolytope) == (i + 2)
for vpolytope in upolytope:
transformed_pre, transformed_post = vpolytope
assert len(transformed_pre) == len(transformed_post) == 3
assert np.allclose(transformed_pre, np.eye(3))
assert np.allclose(transformed_post, np.matmul(plane, weights))
assert len(stub.received_messages) == 1
received = stub.received_messages[0]
assert len(received) == 102
assert received[0].WhichOneof("request_data") == "layer"
assert received[0].layer == fullyconnected_layer.serialize()
assert received[1].layer == relu_layer.serialize()
for i, request in enumerate(received[2:]):
assert request.WhichOneof("request_data") == "upolytope"
assert request.upolytope.space_dimensions == input_dims
assert request.upolytope.subspace_dimensions == 2
assert len(request.upolytope.polytopes) == 1
assert request.upolytope.polytopes[0].num_vertices == 3
assert np.allclose(request.upolytope.polytopes[0].vertices,
planes[i].flatten())
assert np.allclose(request.upolytope.polytopes[0].combinations,
np.eye(3).flatten())
def layer_from_onnx(graph, node):
"""Reads a layer from an ONNX node.
Specs for the ONNX operators are available at:
https://github.com/onnx/onnx/blob/master/docs/Operators.md
"""
# First, we get info about inputs to the layer (including previous
# layer outputs & things like weight matrices).
inputs = node.input
deserialized_inputs = []
deserialized_input_shapes = []
for input_name in inputs:
# We need to find the initializers (which I think are basically
# weight tensors) for the particular input.
initializers = [init for init in graph.initializer
if str(init.name) == str(input_name)]
if initializers:
assert len(initializers) == 1
# Get the weight tensor as a Numpy array and save it.
deserialized_inputs.append(numpy_helper.to_array(initializers[0]))
else:
# This input is the output of another node, so just store the
# name of that other node (we'll link them up later). Eg.
# squeezenet0_conv0_fwd.
deserialized_inputs.append(str(input_name))
# Get metadata about the input (eg. its shape).
infos = [info for info in graph.value_info
if info.name == input_name]
if infos:
# This is an input with a particular shape.
assert len(infos) == 1
input_shape = [d.dim_value
for d in infos[0].type.tensor_type.shape.dim]
deserialized_input_shapes.append(input_shape)
elif input_name == "data":
# This is an input to the entire network, its handled
# separately.
net_input_shape = graph.input[0].type.tensor_type.shape
input_shape = [d.dim_value for d in net_input_shape.dim]
deserialized_input_shapes.append(input_shape)
else:
# This doesn't have any inputs.
deserialized_input_shapes.append(None)
layer = None
# Standardize some of the data shared by the strided-window layers.
if node.op_type in {"Conv", "MaxPool", "AveragePool"}:
# NCHW -> NHWC
input_shape = deserialized_input_shapes[0]
input_shape = [input_shape[2], input_shape[3], input_shape[1]]
strides = list(Network.onnx_ints_attribute(node, "strides"))
pads = list(Network.onnx_ints_attribute(node, "pads"))
# We do not support separate begin/end padding.
assert pads[0] == pads[2]
assert pads[1] == pads[3]
pads = pads[1:3]
# Now, parse the actual layers.
if node.op_type == "Conv":
# We don't support dilations or non-1 groups.
dilations = list(Network.onnx_ints_attribute(node, "dilations"))
assert all(dilation == 1 for dilation in dilations)
group = Network.onnx_ints_attribute(node, "group")
assert not group or group == 1
# biases are technically optional, but I don't *think* anyone uses
# that feature.
assert len(deserialized_inputs) == 3
input_data, filters, biases = deserialized_inputs
# OIHW -> HWIO
filters = filters.transpose((2, 3, 1, 0))
window_data = StridedWindowData(input_shape, filters.shape[:2],
strides, pads, biases.shape[0])
layer = Conv2DLayer(window_data, filters, biases)
elif node.op_type == "Relu":
layer = ReluLayer()
elif node.op_type == "MaxPool":
kernel_shape = Network.onnx_ints_attribute(node, "kernel_shape")
window_data = StridedWindowData(input_shape, list(kernel_shape),
strides, pads, input_shape[2])
layer = MaxPoolLayer(window_data)
elif node.op_type == "AveragePool":
kernel_shape = Network.onnx_ints_attribute(node, "kernel_shape")
window_data = StridedWindowData(input_shape, list(kernel_shape),
strides, pads, input_shape[2])
layer = AveragePoolLayer(window_data)
elif node.op_type == "Gemm":
input_data, weights, biases = deserialized_inputs
alpha = Network.onnx_ints_attribute(node, "alpha")
if alpha:
weights *= alpha
beta = Network.onnx_ints_attribute(node, "beta")
if beta:
biases *= beta
trans_A = Network.onnx_ints_attribute(node, "transA")
trans_B = Network.onnx_ints_attribute(node, "transB")
# We compute (X . W) [+ C].
assert not trans_A
if trans_B:
weights = weights.transpose()
layer = FullyConnectedLayer(weights, biases)
elif node.op_type == "BatchNormalization":
epsilon = Network.onnx_ints_attribute(node, "epsilon")
input_data, scale, B, mean, var = deserialized_inputs
# We don't yet support separate scale/bias parameters, though they
# can be rolled in to mean/var.
assert np.allclose(scale, 1.0) and np.allclose(B, 0.0)
layer = NormalizeLayer(mean, np.sqrt(var + epsilon))
elif node.op_type == "Concat":
layer = list(inputs)
elif node.op_type in {"Dropout", "Reshape", "Flatten"}:
# These are (more-or-less) handled implicitly since we pass around
# flattened activation vectors and only work with testing.
layer = False
else:
raise NotImplementedError
assert len(node.output) == 1
return (inputs[0], node.output[0], layer)
def from_eran(cls, net_file):
"""Helper method to read an ERAN net_file into a Network.
Currently only supports a subset of those supported by the original
read_net_file.py. See an example of the type of network file we're
reading here:
https://files.sri.inf.ethz.ch/eran/nets/tensorflow/mnist/mnist_relu_3_100.tf
This code has been adapted (with heavy modifications) from the ERAN
source code. Each layer has a header line that describes the type of
layer, which is then followed by the weights (if applicable). Note that
some layers are rolled together in ERAN but we do separately (eg.
"ReLU" in the ERAN format corresponds to Affine + ReLU in our
representation).
"""
layers = []
net_file = open(net_file, "r")
while True:
curr_line = net_file.readline()[:-1]
if curr_line in {"Affine", "ReLU", "HardTanh"}:
# Parses a fully-connected layer, possibly followed by
# non-linearity.
# ERAN files use (out_dims, in_dims), we use the opposite.
weights = cls.parse_np_array(net_file).transpose()
biases = cls.parse_np_array(net_file)
if len(layers) > 1 and isinstance(layers[-2], Conv2DLayer):
# When there's an affine after a 2D convolution, ERAN's
# files assume the input is CHW when it's actually HWC. We
# correct that here by permuting the dimensions.
conv_layer = layers[-2]
output_size = weights.shape[-1]
weights = weights.reshape(
(conv_layer.window_data.out_channels,
conv_layer.window_data.out_height(),
conv_layer.window_data.out_width(),
output_size))
weights = weights.transpose(1, 2, 0, 3)
weights = weights.reshape((-1, output_size))
# Add the fully-connected layer.
layers.append(FullyConnectedLayer(weights, biases))
# Maybe add a non-linearity.
if curr_line == "ReLU":
layers.append(ReluLayer())
elif curr_line == "HardTanh":
layers.append(HardTanhLayer())
elif curr_line.startswith("Normalize"):
# Parses a Normalize layer.
means = curr_line.split("mean=")[1].split("std=")[0].strip()
means = cls.parse_np_array(means)
stds = curr_line.split("std=")[1].strip()
stds = cls.parse_np_array(stds)
layers.append(NormalizeLayer(means, stds))
elif curr_line.startswith("Conv2D"):
# Parses a 2D-Convolution layer. The info line looks like:
# ReLU, filters=16, kernel_size=[4, 4], \
# input_shape=[28, 28, 1], stride=[2, 2], padding=0
# But, we can get filters and kernel_size from the actual
# filter weights, so no need to parse that here.
info_line = net_file.readline()[:-1].strip()
activation = info_line.split(",")[0]
stride = cls.parse_np_array(
info_line.split("stride=")[1].split("],")[0] + "]")
input_shape = info_line.split("input_shape=")[1].split("],")[0]
input_shape += "]"
input_shape = cls.parse_np_array(input_shape)
pad = (0, 0)
if "padding=" in info_line:
pad = int(info_line.split("padding=")[1])
pad = (pad, pad)
# (f_h, f_w, i_c, o_c)
filter_weights = cls.parse_np_array(net_file)
# (o_c,)
biases = cls.parse_np_array(net_file)
window_data = StridedWindowData(
input_shape, filter_weights.shape[:2],
stride, pad, filter_weights.shape[3])
layers.append(Conv2DLayer(window_data, filter_weights, biases))
if activation == "ReLU":
layers.append(ReluLayer())
elif activation == "HardTanh":
layers.append(HardTanhLayer())
else:
# As far as I know, all Conv2D layers should have an
# associated activation function in the ERAN format.
raise NotImplementedError
elif curr_line.strip() == "":
break
else:
raise NotImplementedError
return cls(layers)