def run_library_on_remote(self, host: str, output_path: str, library: str,
input_npy: str,
expected_output_npy: str) -> float:
run_and_check(["ssh", f"root@{host}", f"rm -rf ~/automated_testing/*"],
output_path, join(output_path, "clean.err"),
join(output_path, "clean.err"), self)
lib_name = os.path.basename(library)
input_name = os.path.basename(input_npy)
output_name = os.path.basename(expected_output_npy)
run_library_code = "import numpy as np\n"
run_library_code += "from nnlib import NNLib as NNLib\n"
run_library_code += "class testing:\n"
run_library_code += inspect.getsource(self.run_library)
run_library_code += "if __name__ == '__main__':\n"
run_library_code += " t = testing()\n"
run_library_code += f" print(t.run_library('./{lib_name}', './{input_name}', './{output_name}'))\n"
testing_code_name = "testing_code.py"
testing_code_path = join(output_path, testing_code_name)
with open(testing_code_path, "w") as code_file:
code_file.write(run_library_code)
run_and_check([
"scp", library, input_npy, expected_output_npy,
inspect.getfile(NNLib), testing_code_path,
f"root@{host}:~/automated_testing/"
], output_path, join(output_path, "scp.out"),
join(output_path, "scp.err"), self)
remote_output_file = join(output_path, "remote.out")
run_and_check([
"ssh", f"root@{host}",
f"cd ~/automated_testing/; python {testing_code_name}"
],
output_path,
remote_output_file,
join(output_path, "remote.err"),
self,
keep_outputs=True)
with open(remote_output_file, "r") as remote_output_file:
remote_output = remote_output_file.read()
pf = 100.0
try:
pf = float(remote_output)
except (TypeError, ValueError):
pf = 100.0
return pf
def codegen_cpu(self,
model_path,
expected_output_set_name,
prefix,
input_name,
output_npy_name,
cpu_name='x86_64',
hard_quantize=False,
threshold_skipping=False,
use_run_test_script=False,
max_percent_incorrect_values=0.1,
from_npy=False,
need_arm_compiler=False,
cache_dma=False,
use_avx=False,
test_id=0) -> None:
"""Test code for testing code generation for CPU"""
# TODO consider better implementation
if need_arm_compiler:
if shutil.which('arm-linux-gnueabihf-g++') is None:
raise unittest.SkipTest('No arm compiler.')
dir_tags = [str(test_id), prefix, basename(model_path), cpu_name]
dir_tags = dir_tags + ['hq'] if hard_quantize else dir_tags
dir_tags = dir_tags + ['thskip'] if threshold_skipping else dir_tags
output_path = join(self.build_dir, '_'.join(dir_tags))
input_dir_path = os.path.abspath(os.path.join(os.getcwd(), model_path))
input_path = os.path.join(input_dir_path,
'minimal_graph_with_shape.pb')
project_name = 'code_generation'
gp.run(
input_path=input_path,
dest_dir_path=output_path,
project_name=project_name,
activate_hard_quantization=hard_quantize,
threshold_skipping=threshold_skipping,
debug=False,
cache_dma=cache_dma,
)
if cpu_name == 'arm_fpga':
lib_base_name = 'fpga'
elif cpu_name == 'x86_64':
if use_avx:
lib_base_name = 'x86_avx'
else:
lib_base_name = 'x86'
else: # 'aarch64' and 'arm' pass here
lib_base_name = cpu_name
project_dir = os.path.join(output_path, project_name + '.prj')
generated_lib = os.path.join(project_dir,
'libdlk_' + lib_base_name + '.so')
npy_targz = os.path.join(input_dir_path,
expected_output_set_name + '.tar.gz')
run_and_check(
['tar', 'xvzf',
str(npy_targz), '-C',
str(output_path)],
input_dir_path,
join(output_path, "tar_xvzf.out"),
join(output_path, "tar_xvzf.err"),
self,
check_stdout_include=[expected_output_set_name + '/raw_image.npy'])
npy_path = os.path.join(output_path, expected_output_set_name)
input_path = os.path.join(npy_path, input_name)
expected_output_path = os.path.join(npy_path, output_npy_name)
self.assertTrue(os.path.exists(project_dir))
run_and_check(['make', 'clean'], project_dir,
join(output_path, "make_clean.out"),
join(output_path, "make_clean.err"), self)
run_and_check(
['make', 'build', 'ARCH=' + lib_base_name, 'TYPE=dynamic', '-j8'],
project_dir,
join(output_path, "make.out"),
join(output_path, "make.err"),
self,
check_stderr_block=['error: '])
self.assertTrue(os.path.exists(generated_lib))
if not use_run_test_script:
if cpu_name == 'x86_64':
percent_failed = self.run_library(generated_lib, input_path,
expected_output_path)
elif cpu_name == 'arm' or cpu_name == 'arm_fpga':
percent_failed = \
self.run_library_on_remote(FPGA_HOST, output_path, generated_lib, input_path, expected_output_path)
elif cpu_name == 'aarch64':
# Skip run library test, it will run on another test by using saved output files.
self.save_output(test_id, generated_lib, input_path,
expected_output_path)
return
else:
self.fail("Unexpected cpu_name: %s" % cpu_name)
else:
percent_failed = self.run_library_using_script(
generated_lib, input_path, expected_output_path, from_npy)
self.assertTrue(
percent_failed < max_percent_incorrect_values,
msg=
f"Test failed: {percent_failed:.3f}% of the values does not match")
print(
f"Codegen test {prefix}: passed! {100.0 - percent_failed:.3f}% "
f"of the output values are correct\n"
f"[hard quantize == {hard_quantize}, threshold skipping == {threshold_skipping}, cache == {cache_dma}]"
)
def codegen_cpu(self,
model_path,
expected_output_set_name,
prefix,
input_name,
output_npy_name,
cpu_name='x86_64',
hard_quantize=False,
threshold_skipping=False,
use_run_test_script=False,
max_percent_incorrect_values=0.1,
from_npy=False,
need_arm_compiler=False,
cache_dma=False,
use_avx=False,
test_id=0) -> None:
"""Test code for testing code generation for CPU"""
# TODO consider better implementation
if need_arm_compiler:
if shutil.which('arm-linux-gnueabihf-g++') is None:
raise unittest.SkipTest('No arm compiler.')
dir_tags = [str(test_id), prefix, basename(model_path), cpu_name]
dir_tags = dir_tags + ['hq'] if hard_quantize else dir_tags
dir_tags = dir_tags + ['thskip'] if threshold_skipping else dir_tags
output_path = join(self.build_dir, '_'.join(dir_tags))
input_dir_path = os.path.abspath(os.path.join(os.getcwd(), model_path))
input_path = os.path.join(input_dir_path,
'minimal_graph_with_shape.pb')
project_name = 'code_generation'
gp.run(
input_path=input_path,
dest_dir_path=output_path,
project_name=project_name,
activate_hard_quantization=hard_quantize,
threshold_skipping=threshold_skipping,
num_pe=16,
debug=False,
cache_dma=cache_dma,
)
lib_name = 'lib_' + cpu_name
project_dir = os.path.join(output_path, project_name + '.prj')
generated_lib = os.path.join(project_dir, lib_name + '.so')
npy_targz = os.path.join(input_dir_path,
expected_output_set_name + '.tar.gz')
run_and_check(
['tar', 'xvzf',
str(npy_targz), '-C',
str(output_path)],
input_dir_path,
join(output_path, "tar_xvzf.out"),
join(output_path, "tar_xvzf.err"),
self,
check_stdout_include=[expected_output_set_name + '/raw_image.npy'])
npy_path = os.path.join(output_path, expected_output_set_name)
input_path = os.path.join(npy_path, input_name)
expected_output_path = os.path.join(npy_path, output_npy_name)
self.assertTrue(os.path.exists(project_dir))
cmake_use_aarch64 = '-DTOOLCHAIN_NAME=linux_aarch64'
cmake_use_arm = '-DTOOLCHAIN_NAME=linux_arm'
cmake_use_neon = '-DUSE_NEON=1'
cmake_use_fpga = '-DRUN_ON_FPGA=1'
cmake_use_avx = '-DUSE_AVX=1'
cmake_defs = []
if cpu_name == 'aarch64':
cmake_defs += [cmake_use_aarch64, cmake_use_neon]
elif cpu_name == 'arm':
cmake_defs += [cmake_use_arm, cmake_use_neon]
elif cpu_name == 'arm_fpga':
cmake_defs += [cmake_use_arm, cmake_use_neon, cmake_use_fpga]
elif cpu_name == 'x86_64':
if use_avx:
cmake_defs += [cmake_use_avx]
run_and_check(['cmake'] + cmake_defs + ['.'],
project_dir,
join(output_path, "cmake.out"),
join(output_path, "cmake.err"),
self,
check_stdout_include=['Generating done'],
check_stdout_block=['CMake Error'])
run_and_check(['make', 'VERBOSE=1', 'lib', '-j8'],
project_dir,
join(output_path, "make.out"),
join(output_path, "make.err"),
self,
check_stdout_include=['Building'],
check_stderr_block=['error: '])
self.assertTrue(os.path.exists(generated_lib))
if not use_run_test_script:
if cpu_name == 'x86_64':
percent_failed = self.run_library(generated_lib, input_path,
expected_output_path)
elif cpu_name == 'arm' or cpu_name == 'arm_fpga':
percent_failed = \
self.run_library_on_remote(FPGA_HOST, output_path, generated_lib, input_path, expected_output_path)
elif cpu_name == 'aarch64':
# Skip run library test, it will run on another test.
return
else:
self.fail("Unexpected cpu_name: %s" % cpu_name)
else:
percent_failed = self.run_library_using_script(
generated_lib, input_path, expected_output_path, from_npy)
self.assertTrue(
percent_failed < max_percent_incorrect_values,
msg=
f"Test failed: {percent_failed:.3f}% of the values does not match")
print(
f"Codegen test {prefix}: passed! {100.0 - percent_failed:.3f}% "
f"of the output values are correct\n"
f"[hard quantize == {hard_quantize}, threshold skipping == {threshold_skipping}, cache == {cache_dma}]"
)
def test_time_measurement_with_x86(self) -> None:
"""Test code for time measurement on x86."""
model_path = os.path.join('examples', 'classification',
'lmnet_quantize_cifar10')
output_path = self.build_dir
project_name = 'test_binary'
project_dir = os.path.join(output_path, project_name + '.prj')
generated_bin = os.path.join(project_dir, 'lm_x86_64')
input_dir_path = os.path.abspath(os.path.join(os.getcwd(), model_path))
input_path = os.path.join(input_dir_path,
'minimal_graph_with_shape.pb')
debug_data_filename = 'cat.jpg'
compressed_debug_data_path = os.path.join(
input_dir_path, debug_data_filename + '.tar.gz')
debug_data_path = os.path.join(output_path, debug_data_filename)
debug_data_input = os.path.join(debug_data_path,
'000_images_placeholder:0.npy')
debug_data_output = os.path.join(debug_data_path, '133_output:0.npy')
gp.run(
input_path=input_path,
dest_dir_path=output_path,
project_name=project_name,
activate_hard_quantization=False,
threshold_skipping=False,
num_pe=16,
use_tvm=False,
use_onnx=False,
debug=False,
cache_dma=False,
)
self.assertTrue(os.path.exists(project_dir))
run_and_check(['cmake', '.'], project_dir,
join(project_dir, "make.out"),
join(project_dir, "make.err"), self)
run_and_check(['make', 'lm', '-j8'], project_dir,
join(project_dir, "cmake.out"),
join(project_dir, "cmake.err"), self)
self.assertTrue(os.path.exists(generated_bin))
run_and_check(
[
'tar', 'xvzf',
str(compressed_debug_data_path), '-C',
str(output_path)
],
output_path,
join(output_path, "tar_xvzf.out"),
join(output_path, "tar_xvzf.err"),
self,
check_stdout_include=[debug_data_filename + '/raw_image.npy'])
self.assertTrue(os.path.exists(debug_data_input))
self.assertTrue(os.path.exists(debug_data_output))
run_and_check([
str(generated_bin),
str(debug_data_input),
str(debug_data_output)
],
project_dir,
join(project_dir, "elf.out"),
join(project_dir, "elf.err"),
self,
check_stdout_include=['TotalRunTime '])
print(f"Binary time-measurement test : passed!")
def test_arm_binary(self) -> None:
"""Test code for testing arm binary.
This code assumes to be executed only on the ARM emulator environement."""
cxx = 'arm-linux-gnueabihf-g++'
cxx_path = shutil.which(cxx)
if cxx_path is None:
print('No arm compiler nor library. Quit testing.')
raise unittest.SkipTest('No arm compiler nor library')
else:
arm_path = Path(cxx_path).parent.parent
qemu = 'qemu-arm'
# only works on Jenkins server
arm_lib = arm_path.joinpath('arm-linux-gnueabihf').joinpath('libc')
output_path = join(self.build_dir, get_func_name())
model_path = os.path.join('examples', 'classification',
'lmnet_quantize_cifar10')
input_dir_path = os.path.abspath(os.path.join(os.getcwd(), model_path))
input_path = os.path.join(input_dir_path,
'minimal_graph_with_shape.pb')
project_name = 'arm_binary'
# code generation
gp.run(
input_path=str(input_path),
dest_dir_path=str(output_path),
project_name=str(project_name),
activate_hard_quantization=True,
threshold_skipping=False,
num_pe=16,
use_tvm=False,
use_onnx=False,
debug=False,
cache_dma=False,
)
cpu_name = 'arm'
bin_name = 'lm_' + cpu_name
project_dir = Path(output_path).joinpath(project_name + '.prj')
generated_bin = project_dir.joinpath(bin_name + '.elf')
command0 = ['make', 'lm_arm', '-j8']
run_and_check(command0,
project_dir,
join(output_path, "command0.out"),
join(output_path, "command0.err"),
self,
check_stdout_include=['g++'],
check_stdout_block=['error: '])
self.assertTrue(os.path.exists(generated_bin))
# prepare debug data
debug_data_filename = 'cat.jpg'
compressed_debug_data_path = os.path.join(
input_dir_path, debug_data_filename + '.tar.gz')
debug_data_path = os.path.join(output_path, debug_data_filename)
debug_data_input = os.path.join(debug_data_path,
'000_images_placeholder:0.npy')
debug_data_output = os.path.join(debug_data_path, '133_output:0.npy')
run_and_check(
[
'tar', 'xvzf',
str(compressed_debug_data_path), '-C',
str(output_path)
],
output_path,
join(output_path, "tar_xvzf.out"),
join(output_path, "tar_xvzf.err"),
self,
check_stdout_include=[debug_data_filename + '/raw_image.npy'])
self.assertTrue(os.path.exists(debug_data_input))
self.assertTrue(os.path.exists(debug_data_output))
command1 = [qemu, '-L', str(arm_lib)
] if shutil.which(qemu) is not None else []
command1 += [
str(generated_bin),
str(debug_data_input),
str(debug_data_output)
]
print("Running ", command1)
run_and_check(command1,
project_dir,
join(output_path, "command1.out"),
join(output_path, "command1.err"),
self,
check_stdout_include=['succeeded'])
print(f"arm binary test passed!")
def test_arm_specific_code(self) -> None:
"""Test code for testing arm-specific code.
This code assumes to be executed only on the ARM emulator environement."""
cxx = 'arm-linux-gnueabihf-g++'
qemu = 'qemu-arm'
# only works on Jenkins server
arm_lib = '/opt/x-tools/arm-unknown-linux-gnueabihf/arm-unknown-linux-gnueabihf/sysroot/'
if shutil.which(cxx) is None or (shutil.which(qemu) is not None
and not os.path.exists(arm_lib)):
print('No arm compiler nor library. Quit testing.')
raise unittest.SkipTest('No arm compiler nor library')
output_path = join(self.build_dir, get_func_name())
model_path = os.path.join('examples', 'classification',
'lmnet_quantize_cifar10')
input_path = os.path.join(
os.path.abspath(os.path.join(os.getcwd(), model_path)),
'minimal_graph_with_shape.pb')
project_name = 'arm_specific'
# code generation
gp.run(
input_path=input_path,
dest_dir_path=output_path,
project_name=project_name,
activate_hard_quantization=True,
threshold_skipping=False,
num_pe=16,
use_tvm=False,
use_onnx=False,
debug=False,
cache_dma=False,
)
cpu_name = 'arm'
lib_name = 'test_' + cpu_name
project_dir = os.path.join(output_path, project_name + '.prj')
generated_bin = os.path.join(project_dir, lib_name + '.elf')
flags = [
'-I./include', '-std=c++0x', '-O3', '-D__USE_PNG__',
'-mcpu=cortex-a9', '-mfpu=neon', '-mthumb', '-s', '-static'
]
cxxflags = ['-D__ARM__']
# TODO it's better to fix makefile
command0 = [cxx] + cxxflags + flags
commands = [
command0 + ['-c', 'src/pack2b_neonv7.S'],
command0 + ['-c', 'src/pack_input_to_qwords.cpp'],
command0 + ['-c', 'src/time_measurement.cpp'],
command0 + ['-c', 'mains/test_arm_main.cpp'], command0 + [
'pack2b_neonv7.o', 'test_arm_main.o', 'time_measurement.o',
'pack_input_to_qwords.o', '-lpthread', '-o', generated_bin
]
]
for i, command in enumerate(commands):
run_and_check(command, project_dir,
join(output_path, "command0-" + str(i) + ".out"),
join(output_path, "command0-" + str(i) + ".err"),
self)
self.assertTrue(os.path.exists(generated_bin))
command1 = [qemu, '-L', arm_lib
] if shutil.which(qemu) is not None else []
command1 += [generated_bin]
print("Running ", command1)
run_and_check(command1,
project_dir,
join(output_path, "command1.out"),
join(output_path, "command1.err"),
self,
check_stdout_include=["Succeeded"])
print(f"arm-specific code test passed!")
