def test_metric(self):
""" Metric test
"""
ctx = beatmup.Context()
metric = beatmup.Metric()
import numpy
# generate random inputs
array1 = numpy.random.uniform(-0.5, 0.5,
(256, 256, 3)).astype(numpy.float32)
array2 = numpy.random.uniform(-0.5, 0.5,
(256, 256, 3)).astype(numpy.float32)
diff = (array1 - array2).reshape(-1)
img1 = beatmup.Bitmap(ctx, array1)
img2 = beatmup.Bitmap(ctx, array2)
metric.set_bitmaps(img1, img2)
# check L1
metric.set_norm(metric.Norm.L1)
ctx.perform_task(metric)
self.assertAlmostEqual(numpy.linalg.norm(diff, 1), metric.get_result(),
1)
# check L2
metric.set_norm(metric.Norm.L2)
ctx.perform_task(metric)
self.assertAlmostEqual(numpy.linalg.norm(diff, 2), metric.get_result(),
2)
# check PSNR
psnr = 10 * numpy.log10(1 / numpy.mean(diff**2))
self.assertAlmostEqual(beatmup.Metric.psnr(img1, img2), psnr, 5)
def test_image_sampler(self):
""" ImageSampler test
"""
if VERBOSE: print('---- ImageSampler test...')
# generate input image
input_image = make_random_image((48, 32))
center_crop = input_image[8:-8,:,:]
# set up a test model
ref_model = tf.keras.models.Sequential([
tf.keras.layers.Input(input_image.shape),
tf.keras.layers.Conv2D(8, 1,
name='conv',
strides=1,
kernel_initializer='random_normal',
bias_initializer='random_normal',
use_bias=False),
tf.keras.layers.Activation(beatmup_keras.brelu1)
])
# get "test id" to add as prefix to model layers
global model_ctr
test_id = 'test' + str(model_ctr)
# convert model
ctx = beatmup.Context()
model, model_data = beatmup_keras.export_model(ref_model, ctx, prefix=test_id + '__')
# run inference on a cropped input
inference = beatmup.nnets.InferenceTask(model, model_data)
inference.connect(beatmup.Bitmap(ctx, center_crop), model.get_first_operation())
model.add_output(model.get_last_operation())
ctx.perform_task(inference)
ref_output = model.get_output_data(model.get_last_operation())
# add a preprocessing layer
model, model_data = beatmup_keras.export_model(ref_model, ctx, prefix=test_id + '__')
image_sampler = beatmup.nnets.ImageSampler(test_id + "__preprocessing", center_crop.shape[:2])
first_op = model.get_first_operation()
model.add_operation(first_op.name, image_sampler)
model.add_connection(image_sampler.name, first_op.name)
# run on full input
inference = beatmup.nnets.InferenceTask(model, model_data)
inference.connect(beatmup.Bitmap(ctx, input_image), model.get_first_operation())
model.add_output(model.get_last_operation())
ctx.perform_task(inference)
test_output = model.get_output_data(model.get_last_operation())
self.assertTrue(numpy.all(ref_output == test_output))
# export the test
export_test(test_id, self.test_image_sampler.__doc__, model_data, model, input_image, ref_output, 0.004)
def test_multiply_adds_and_texel_fetches(self):
""" Tests multiply-adds and texel fetches counting
"""
# generate input image
input_image = make_random_image((32, 32))
# set up a test model
model = tf.keras.models.Sequential([
tf.keras.layers.Input((32, 32, 3)),
tf.keras.layers.Conv2D(16, kernel_size=3, use_bias=False),
tf.keras.layers.Activation(beatmup_keras.brelu6),
tf.keras.layers.Conv2D(32, kernel_size=3, groups=4, use_bias=False),
tf.keras.layers.Activation(beatmup_keras.brelu6),
tf.keras.layers.MaxPooling2D(3, strides=1)
])
# prepare model
ctx = beatmup.Context()
test_model, test_data = beatmup_keras.export_model(model, ctx)
inference = beatmup.nnets.InferenceTask(test_model, test_data)
inference.connect(beatmup.Bitmap(ctx, input_image), test_model.get_first_operation())
test_model.add_output(test_model.get_last_operation())
ctx.perform_task(inference)
# check
self.assertEqual(test_model.count_multiply_adds(), 30*30*16*3*3*3 + 28*28*32*3*3*4 + 0)
self.assertEqual(test_model.count_texel_fetches(), 30*30*16*3*3//4 + 28*28*32*3*3//4 + 26*26*32*3*3//4)
def replay_tests():
""" Reads a file with test data and replays the tests
"""
# open file
datafile = beatmup.ChunkFile(test_export_filename)
datafile.open()
# prepare things
ctx = beatmup.Context()
# loop till tests
i = 1
prefix = lambda i: 'test' + str(i)
while datafile.chunk_exists(prefix(i)):
# get data
datafile.open()
test_title = bytes.decode(datafile[prefix(i)])
model_code = bytes.decode(datafile[prefix(i) + ':model'])
input_image_shape = numpy.frombuffer(datafile[prefix(i) + ':input_shape'], dtype=numpy.int32)
input_image = numpy.frombuffer(datafile[prefix(i) + ':input'], dtype=numpy.uint8).reshape(input_image_shape)
ref_output = numpy.frombuffer(datafile[prefix(i) + ':gt'], dtype=numpy.float32)
threshold, = struct.unpack('f', datafile[prefix(i) + ':threshold'])
datafile.close()
print('#%02d: %s' % (i, test_title))
# restore model
model = beatmup.nnets.DeserializedModel(ctx, model_code)
# build inference task
inference = beatmup.nnets.InferenceTask(model, datafile)
# connect input
inference.connect(beatmup.Bitmap(ctx, input_image), model.get_first_operation())
# connect output if not softmax; softmax has no outputs
head = model.get_last_operation()
softmax = 'softmax' in head.name
if not softmax:
model.add_output(head)
# run inference
ctx.perform_task(inference)
# get data
test_output = numpy.asarray(head.get_probabilities()) if softmax else model.get_output_data(head)
# get ground truth data
ref_output = ref_output.reshape(test_output.shape)
# check error and print things
error = numpy.max(numpy.abs(test_output - ref_output))
print(' Error: %0.4f for mapping %s to %s' % (error, input_image.shape, test_output.shape))
assert error < threshold
i += 1
def test_rotation(self):
""" ImageSampler rotation test
"""
if VERBOSE: print('---- ImageSampler rotation test...')
# generate input image
input_image = make_random_image((48, 32))
center_crop = input_image[8:-8,:,:]
# set up a test model
ref_model = tf.keras.models.Sequential([
tf.keras.layers.Input(input_image.shape),
tf.keras.layers.Conv2D(4, 1,
name='conv',
strides=1,
kernel_initializer='random_normal',
bias_initializer='random_normal',
use_bias=False),
tf.keras.layers.Activation(beatmup_keras.brelu1)
])
# convert model
ctx = beatmup.Context()
model, model_data = beatmup_keras.export_model(ref_model, ctx)
# add a preprocessing layer
image_sampler = beatmup.nnets.ImageSampler('sampler', center_crop.shape[:2])
first_op = model.get_first_operation()
model.add_operation(first_op.name, image_sampler)
model.add_connection(image_sampler.name, first_op.name)
# run inference on a cropped input
inference = beatmup.nnets.InferenceTask(model, model_data)
inference.connect(beatmup.Bitmap(ctx, center_crop), model.get_first_operation())
model.add_output(model.get_last_operation())
ctx.perform_task(inference)
ref_output = model.get_output_data(model.get_last_operation())
# rotate and test
for i in range(4):
image_sampler.rotation = i
ctx.perform_task(inference)
test_output = model.get_output_data(model.get_last_operation())
self.assertTrue(numpy.all(ref_output == numpy.rot90(test_output, i)))
def wrapped(self, *args, **kwargs):
input_image, ref_model = func(self, *args, **kwargs)
# get "test id" to add as prefix to model layers
global model_ctr
test_id = 'test' + str(model_ctr)
# compute reference output
ref_model.compile()
ref_output = ref_model.predict(make_tensorflow_batch(input_image))[0]
# convert model
ctx = beatmup.Context()
test_model, model_data = beatmup_keras.export_model(ref_model, ctx, prefix=test_id + '__')
# init inference task
inference = beatmup.nnets.InferenceTask(test_model, model_data)
# connect input
inference.connect(beatmup.Bitmap(ctx, input_image), test_model.get_first_operation())
# connect output if not softmax; softmax has no outputs
head = test_model.get_last_operation()
softmax = 'softmax' in head.name
if not softmax:
test_model.add_output(head)
# run inference
ctx.perform_task(inference)
# get data
test_output = numpy.asarray(head.get_probabilities()) if softmax else test_model.get_output_data(head)
# compare
error = numpy.max(numpy.abs(test_output - ref_output))
if VERBOSE: print('Error: %0.4f for %d layers mapping %s to %s' % (error, len(ref_model.layers), input_image.shape, test_output.shape))
del ctx
# export
export_test(test_id, func.__doc__, model_data, test_model, input_image, ref_output, error_threshold)
# assert
self.assertLess(error, error_threshold)
# initialize context
ctx = beatmup.Context()
# crete a resampler
resampler = beatmup.BitmapResampler(ctx)
# choose resampling algorithm
resampler.mode = beatmup.BitmapResampler.CONVNET
# get the input shape
h, w, c = image.shape
assert c == 3, "A three-channel input image is expected"
# set up output bitmap
output = beatmup.Bitmap(ctx, numpy.zeros((2 * h, 2 * w, c), numpy.uint8))
# feed the resampler with input and output
resampler.input = beatmup.Bitmap(ctx, image)
resampler.output = output
# run the resampling task
ctx.perform_task(resampler)
# store the result to file
result = cv2.imwrite(output_filename, numpy.array(output, copy=False))
if result:
print("Result is written to", output_filename)
exit(0)
else:
print("Cannot write to", output_filename)
def test_serialization(self):
""" Tests model serialization and reconstruction
"""
if VERBOSE: print('---- Serialization...')
# generate input image
input_image = make_random_image((32, 32))
# set up a test model
input = tf.keras.layers.Input(input_image.shape)
x = tf.keras.layers.Conv2D(32, 3,
name='conv_1',
strides=2,
kernel_initializer='random_normal',
bias_initializer='random_normal',
use_bias=False)(input)
x = residual = tf.keras.layers.Activation(beatmup_keras.brelu6)(x)
x = tf.keras.layers.DepthwiseConv2D(3,
name='depthwise_conv_2',
strides=1,
kernel_initializer='random_normal',
bias_initializer='random_normal',
padding='same',
use_bias=True)(x)
x = tf.keras.layers.Activation(beatmup_keras.brelu6)(x)
x = tf.keras.layers.Conv2D(32, 1,
name='pointwise_conv_2',
strides=1,
kernel_initializer='random_normal',
bias_initializer='random_normal',
use_bias=True)(x)
x = tf.keras.layers.Add(name="add_residual_1")([x, residual])
x = tf.keras.layers.Activation(beatmup_keras.brelu1)(x)
x = tf.keras.layers.MaxPooling2D(2)(x)
x = tf.keras.layers.Conv2D(64, 1,
name='pointwise_conv',
kernel_initializer='random_normal',
bias_initializer='random_normal',
use_bias=True)(x)
x = tf.keras.layers.ReLU(max_value=2.0)(x)
x = residual = beatmup_keras.Shuffle(2)(x)
x = tf.keras.layers.DepthwiseConv2D(3,
name='depthwise_conv_3',
strides=1,
kernel_initializer='random_normal',
bias_initializer='random_normal',
padding='same',
use_bias=True)(x)
x = tf.keras.layers.Activation(beatmup_keras.brelu6)(x)
x = tf.keras.layers.Conv2D(64, 1,
name='pointwise_conv_3',
strides=1,
kernel_initializer='random_normal',
bias_initializer='random_normal',
use_bias=True)(x)
x = tf.keras.layers.Add(name="add_residual_2")([x, residual])
x = tf.keras.layers.ReLU(max_value=2.0)(x)
x = tf.keras.layers.GlobalAveragePooling2D()(x)
x = tf.keras.layers.Dense(40)(x)
x = tf.keras.layers.Softmax()(x)
# make a model
ref_model = tf.keras.models.Model(inputs=input, outputs=x)
ref_model.compile()
ref_output = ref_model.predict(make_tensorflow_batch(input_image))[0]
# convert model
ctx = beatmup.Context()
model, model_data = beatmup_keras.export_model(ref_model, ctx)
# run inference
inference = beatmup.nnets.InferenceTask(model, model_data)
inference.connect(beatmup.Bitmap(ctx, input_image), model.get_first_operation())
ctx.perform_task(inference)
output = model.get_last_operation().get_probabilities()
# print stuff
error = numpy.max(numpy.abs(output- ref_output))
if VERBOSE: print("Error: %0.4f for %d layers mapping %s to %s" % (error, len(ref_model.layers), input_image.shape, len(output)))
# serialize
serial = model.serialize()
# reconstruct
reconstructed_model = beatmup.nnets.DeserializedModel(ctx, serial)
# run inference of the reconstructed model
inference_rec = beatmup.nnets.InferenceTask(reconstructed_model, model_data)
inference_rec.connect(beatmup.Bitmap(ctx, input_image), reconstructed_model.get_first_operation())
ctx.perform_task(inference_rec)
output_rec = model.get_last_operation().get_probabilities()
# compare
self.assertEqual(output, output_rec)
# From the inference standpoint, it is somehow equivalent to have a softmax layer at the end, in order to get classification probabilities.
# Add a Softmax layer to the converted model, and connect it to the last dense layer:
test_model.append(beatmup.nnets.Softmax('softmax'))
test_model.add_connection('dense', 'softmax')
# Initialize inference task
inference = beatmup.nnets.InferenceTask(test_model, test_model_data)
# Run the inference on CIFAR100 test set in a loop over all the images
print('===== Running inference on the test set...')
total = len(test_labels) # total number of images
score = 0 # number of correctly classified images
for i in range(total):
# Get the image
image = numpy.copy(test_images[i], 'C')
image = beatmup.Bitmap(ctx, image)
# Connect it to the model
inference.connect(image, test_model.get_first_operation())
# Run the inference: this is where Beatmup does the job
ctx.perform_task(inference)
# Get the class probabilities and the class label
probabilities = test_model.get_last_operation().get_probabilities()
prediction = numpy.argmax(probabilities)
# Score a point if the prediction matches the ground truth
if prediction == test_labels[i]:
score += 1