def benchmarkBinary(benchmark, binary):
binpath = os.path.join("bin", binary)
info = binary.split('_')
primitive = '_'.join(info[:2])
scheme = '_'.join(info[2:-2])
implementation = info[-2]
if utils.m4ignore(primitive, scheme, implementation):
return
if len(sys.argv) > 1 and scheme not in sys.argv[1:]:
return
result = utils.m4run(binpath)
if 'ERROR KEYS' in result:
print("")
print("!!! KEY MISMATCH DURING BENCHMARKING !!!")
print(" This could indicate illegal stack usage,")
print(" leading to errors when measurement interrupts occur.")
print("")
print(" .. exiting with errors!")
sys.exit(1)
timestamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d%H%M%S')
filename = os.path.join('benchmarks/', benchmark, primitive, scheme, implementation, timestamp)
os.makedirs(os.path.dirname(filename), exist_ok=True)
with open(filename, 'w') as f:
f.write(result.strip())
print(" .. wrote benchmarks!")
def doTest(binary):
binpath = os.path.join("bin", binary)
info = binary.split('_')
primitive = '_'.join(info[:2])
scheme = '_'.join(info[2:-2])
implementation = info[-2]
if m4ignore(primitive, scheme, implementation):
return
if len(sys.argv) > 1 and scheme not in sys.argv[1:]:
return
print("Flashing {}..".format(binpath))
subprocess.run(["st-flash", "write", binpath, "0x8000000"],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
print("Flashed, now running tests...")
state = 'waiting'
marker = b''
# This parses test vector output starting with a number of leading '=',
# and expects a hashtag '#' after the test vector output.
while True:
x = dev.read()
if x == b'' and state == 'waiting':
print("timed out while waiting for the markers")
doTest(binary)
return
if state == 'waiting':
if x == b'=':
marker += x
continue
# If we saw at least 5 equal signs, assume we've probably started
elif marker.count(b'=') > 5:
state = 'beginning'
vector = []
print(" .. found output marker..")
if state == 'beginning':
if x == b'=':
continue
else:
state = 'reading'
elif state == 'reading':
if x == b'#':
break
else:
vector.append(x)
print("Testing if tests were successful..")
contents = b''.join(vector).decode('utf-8').strip()
# can we find a nicer way of checking if tests ran correctly?
if contents.count("ERROR") != 0 or contents.count("OK") != 30:
print("FAILED!")
else:
print("passed.")
def benchmarkBinary(benchmark, binary):
binpath = os.path.join("bin", binary)
info = binary.split('_')
primitive = '_'.join(info[:2])
scheme = '_'.join(info[2:-2])
implementation = info[-2]
if m4ignore(primitive, scheme, implementation):
return
if len(sys.argv) > 1 and scheme not in sys.argv[1:]:
return
print("Flashing {}..".format(binpath))
subprocess.run(["st-flash", "write", binpath, "0x8000000"],
stdout=sys.stdout.buffer,
stderr=sys.stdout.buffer)
print("Flashed, now running benchmarks..".format(binary))
state = 'waiting'
marker = b''
# This parses test vector output starting with a number of leading '=',
# and expects a hashtag '#' after the test vector output.
while True:
x = dev.read()
if x == b'' and state == 'waiting':
print("timed out while waiting for the markers")
benchmarkBinary(benchmark, binary)
return
if state == 'waiting':
if x == b'=':
marker += x
continue
# If we saw at least 5 equal signs, assume we've probably started
elif marker.count(b'=') > 5:
state = 'beginning'
vector = []
print(" .. found output marker..")
if state == 'beginning':
if x == b'=':
continue
else:
state = 'reading'
elif state == 'reading':
if x == b'#':
break
else:
vector.append(x)
timestamp = datetime.datetime.fromtimestamp(
time.time()).strftime('%Y%m%d%H%M%S')
filename = os.path.join('benchmarks/', benchmark, primitive, scheme,
implementation, timestamp)
os.makedirs(os.path.dirname(filename), exist_ok=True)
with open(filename, 'w') as f:
f.write(b''.join(vector).decode('utf-8').strip())
print(" .. wrote benchmarks!")
def doTest(binary):
binpath = os.path.join("bin", binary)
info = binary.split('_')
primitive = '_'.join(info[:2])
scheme = '_'.join(info[2:-2])
implementation = info[-2]
if utils.m4ignore(primitive, scheme, implementation):
return
if len(sys.argv) > 1 and scheme not in sys.argv[1:]:
return
result = utils.m4run(binpath)
print("Testing if tests were successful..")
contents = result.strip()
# can we find a nicer way of checking if tests ran correctly?
if contents.count("ERROR") != 0 or contents.count("OK") != 30:
print("FAILED!")
else:
print("passed.")
def benchmarkBinary(benchmark, binary):
binpath = os.path.join("bin", binary)
info = binary.split('_')
primitive = '_'.join(info[:2])
scheme = '_'.join(info[2:-2])
implementation = info[-2]
if m4ignore(primitive, scheme, implementation):
return
# if len(sys.argv) > 1 and scheme not in sys.argv[1:]:
# return
# subprocess.run(["st-flash", "write", binpath, "0x8000000"],
# stdout=sys.stdout.buffer, stderr=sys.stdout.buffer)
if(to_flash_or_not == 0):
print("Flashing {}..".format(binpath))
str = "program " + binpath + " 0x08000000 verify reset exit"
subprocess.run(["openocd", "-f", "interface/stlink-v2.cfg", "-f", "target/stm32f4x.cfg", "-c", str])
print("Flashed, now running benchmarks..".format(binary))
# state = 'waiting'
# marker = b''
for j in range(prob_parameter_low,prob_parameter_high,prob_parameter_step):
for i in range(limit_parameter_low,limit_parameter_high,limit_parameter_step):
print("Limit parameter: " + repr(i))
print("Prob parameter: " + repr(j))
a = 0
sent = 0
written = 0
received_marker = 0
state = 'waiting'
marker = b''
while True:
if(received_marker == 0):
x = dev.read()
if x == b'' and state == 'waiting':
print("timed out while waiting for the markers")
benchmarkBinary(benchmark, binary)
return
if state == 'waiting':
if x == b'=':
marker += x
if(marker.count(b'=') > 5):
received_marker = 1
x = b''.decode('utf-8')
continue
# If we saw at least 5 equal signs, assume we've probably started
elif received_marker == 1:
state = 'reading'
vector = []
print(" .. found output marker..")
elif state == 'reading':
if x == b'#':
break
else:
if(sent == 0):
if(i < 0):
limit_sign = bytes([1])
limit_param = bytes([abs(i)&0xFF])
else:
limit_sign = bytes([0])
limit_param = bytes([i&0xFF])
prob_param = bytes([j&0xFF])
# print(chr(i&0xFF))
# print(chr(i&0xFF))
# print(chr(j&0xFF))
# dev.write(limit_sign.encode('utf-8'))
# dev.write(limit_param.encode('utf-8'))
# dev.write(prob_param.encode('utf-8'))
dev.write(limit_sign)
dev.write(limit_param)
dev.write(prob_param)
# x = dev.read(3)
# print(x)
# x = dev.read(3)
# print(x)
sent = 1
received_marker = 0
if(written == 1):
vector.append(x)
written = 1
# timestamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y%m%d%H%M%S')
ii = "iteration_loop_count_%d_%d" %(i,j)
filename = os.path.join('benchmarks/', benchmark, primitive, scheme, implementation, ii)
os.makedirs(os.path.dirname(filename), exist_ok=True)
with open(filename, 'w') as f:
f.write(b''.join(vector).decode('utf-8').strip())
print(" .. wrote benchmarks!")
print(" writes the resulting output to the testvectors directory. It then")
print(" checks if these are internally consistent and match the test vectors")
print(" when running the reference code locally.")
for binary in binaries + binaries_host:
info = binary.split('_')
primitive = '_'.join(info[:2])
scheme = '_'.join(info[2:-2])
impl = info[-2]
if len(sys.argv) > 1 and scheme not in sys.argv[1:]:
continue
# if this is a binary that needs to be ran on the board:
if binary[-4:] == '.bin':
if m4ignore(primitive, scheme, impl):
continue
binpath = os.path.join("bin", binary)
print("Flashing {}..".format(binpath))
subprocess.run(["st-flash", "write", binpath, "0x8000000"],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
print("Flashed, now computing test vectors..".format(binary))
state = 'waiting'
marker = b''
# This parses test vector output starting with a number of leading '=',
# and expects a hashtag '#' after the test vector output.
while True:
print(" writes the resulting output to the testvectors directory. It then")
print(" checks if these are internally consistent and match the test vectors")
print(" when running the reference code locally.")
for binary in binaries + binaries_host:
info = binary.split('_')
primitive = '_'.join(info[:2])
scheme = '_'.join(info[2:-2])
impl = info[-2]
if len(sys.argv) > 1 and scheme not in sys.argv[1:]:
continue
# if this is a binary that needs to be ran on the board:
if binary[-4:] == '.bin':
if utils.m4ignore(primitive, scheme, impl):
continue
binpath = os.path.join("bin", binary)
results = utils.m4run(binpath)
filename = os.path.join('testvectors/', primitive, scheme, impl)
os.makedirs(os.path.dirname(filename), exist_ok=True)
with open(filename, 'w') as f:
f.write(results.lstrip())
else:
binpath = os.path.join("bin-host", binary)
print("Running {}..".format(binpath))
filename = os.path.join('testvectors/', primitive, scheme, "host")
os.makedirs(os.path.dirname(filename), exist_ok=True)
with open(filename, 'w') as f:
subprocess.run([binpath], stdout=f, stderr=subprocess.DEVNULL)