def PreTestSetup(self):
"""Sets up test before running."""
# This sed command makes shell scripts compatible wiht android shell.
sed_pattern = [
's?/bin/echo?echo?',
's?#!/bin/sh?#!/system/bin/sh?',
's?#!/bin/bash?#!/system/bin/sh?'
]
sed_cmd = 'sed %s' % ' '.join(
['-i -e ' + ('"%s"' % p) for p in sed_pattern])
# This grep command is used to identify shell scripts.
grep_pattern = [
'bin/sh',
'bin/bash'
]
grep_cmd = 'grep -l %s' % ' '.join(
['-e ' + ('"%s"' % p) for p in grep_pattern])
# This applies sed_cmd to every shell script.
cmd = 'find %s -type f | xargs %s | xargs %s' % (
config.KSFT_DIR, grep_cmd, sed_cmd)
result = self._shell.Execute(cmd)
asserts.assertFalse(
any(result[const.EXIT_CODE]),
"Error: pre-test setup failed.")
def _runBenchmark(self, bits):
"""Runs the native binary and parses its result.
Args:
bits: integer (32 or 64), the bitness of the binary to run.
Returns:
dict, the benchmarking result converted from native binary's JSON
output.
"""
logging.info("Start %d-bit hwbinder latency test with HIDL mode=%s",
bits, self.hidl_hal_mode)
binary = "/data/local/tmp/%s/libhwbinder_latency%s" % (bits, bits)
min_cpu, max_cpu = self._cpu_freq.GetMinAndMaxCpuNo()
iterations = 1000 // (max_cpu - min_cpu)
results = self.dut.shell.one.Execute([
"chmod 755 %s" % binary,
"VTS_ROOT_PATH=/data/local/tmp " \
"LD_LIBRARY_PATH=/system/lib%s:/data/local/tmp/%s/hw:"
"/data/local/tmp/%s:$LD_LIBRARY_PATH "
"%s -raw_data -pair %d -i %d -m %s" % (bits, bits, bits,
binary, max_cpu - min_cpu, iterations,
self.hidl_hal_mode.encode("utf-8"))])
# Parses the result.
asserts.assertEqual(len(results[const.STDOUT]), 2)
logging.info("stderr: %s", results[const.STDERR][1])
logging.info("stdout: %s", results[const.STDOUT][1])
asserts.assertFalse(any(results[const.EXIT_CODE]),
"testRunBenchmark%sBit failed." % (bits))
json_result = json.loads(results[const.STDOUT][1])
asserts.assertTrue(json_result[self._INHERITANCE] == "PASS",
"Scheduler does not support priority inheritance.")
return json_result
def testSerialNotEqual(self):
'''Checks serial number from two device not being equal.'''
command = 'getprop | grep ro.serial'
res1 = self.shell1.Execute(command)
res2 = self.shell2.Execute(command)
asserts.assertFalse(
any(res1[const.EXIT_CODE]),
'command for device 1 failed: %s' % res1) # checks the exit code
asserts.assertFalse(
any(res2[const.EXIT_CODE]),
'command for device 2 failed: %s' % res2) # checks the exit code
def getSerial(output):
'''Get serial from getprop query'''
return output.strip().split(' ')[-1][1:-1]
serial1 = getSerial(res1[const.STDOUT][0])
serial2 = getSerial(res2[const.STDOUT][0])
logging.info('Serial number of device 1: %s', serial1)
logging.info('Serial number of device 2: %s', serial2)
asserts.assertNotEqual(
serial1, serial2,
'serials from two devices should not be the same')
def testWriteTooLarge(self):
"""Test writing more than queue capacity. """
write_data = self.GetRandomIntegers(2049)
# Write overflows the capacity of the queue.
asserts.assertFalse(
self._queue1_writer.write(write_data, 2049),
"Writer should fail since there are too many items to write.")
def AssertShellCommandSuccess(command_results, num_of_commands):
'''Check shell command result with assertions.
Given a shell command output, this command checks several things:
1. result is not None
2. result is not empty
3. number of results is consistant with number of commands
4. there is no error message on STDERR
5. return code of commands are all 0
Args:
command_results: dict, shell command results
num_of_commands: int, number of commands
'''
asserts.assertTrue(command_results is not None,
'command result cannot be None')
asserts.assertEqual(len(command_results), 3, 'command result is empty')
for item in command_results:
asserts.assertEqual(
len(command_results[item]), num_of_commands,
'number of command result is not %s: %s' % (num_of_commands,
command_results))
asserts.assertFalse(
any(command_results[const.STDERR]),
'received error message from stderr: %s' % command_results)
asserts.assertFalse(
any(command_results[const.EXIT_CODE]),
'received non zero return code: %s' % command_results)
def GetHalServiceName(shell, hal, bitness="64", run_as_compliance_test=False):
"""Determine whether to run a VTS test against a HAL and get the service
names of the given hal if determine to run.
Args:
shell: the ShellMirrorObject to execute command on the device.
hal: string, the FQName of a HAL service, e.g.,
[email protected]::IFoo
bitness: string, the bitness of the test.
run_as_compliance_test: boolean, whether it is a compliance test.
Returns:
a boolean whether to run the test against the given hal.
a set containing all service names for the given HAL.
"""
cmd = VTS_TESTABILITY_CHECKER_64
if bitness == "32":
cmd = VTS_TESTABILITY_CHECKER_32
if run_as_compliance_test:
cmd += " -c "
cmd += " -b " + bitness + " " + hal
cmd_results = shell.Execute(str(cmd))
asserts.assertFalse(
any(cmd_results[const.EXIT_CODE]),
"Failed to run vts_testability_checker. Error: %s" %
cmd_results[const.STDERR][0])
result = json.loads(cmd_results[const.STDOUT][0])
if str(result['Testable']).lower() == "true":
return True, set(result['instances'])
else:
return False, ()
def RunTestcase(self, testcase):
"""Runs the given testcase and asserts the result.
Args:
testcase: a LinuxKselftestTestcase object, specifies which
test case to run.
"""
if not testcase:
asserts.skip("Test is not supported on this abi.")
chmod_cmd = "chmod -R 755 %s" % path_utils.JoinTargetPath(
config.KSFT_DIR, testcase.testsuite)
cd_cmd = "cd %s" % path_utils.JoinTargetPath(
config.KSFT_DIR, testcase.testsuite)
cmd = [
chmod_cmd,
"%s && %s" % (cd_cmd, testcase.test_cmd)
]
logging.info("Executing: %s", cmd)
result = self._shell.Execute(cmd)
logging.info("EXIT_CODE: %s:", result[const.EXIT_CODE])
asserts.assertFalse(
any(result[const.EXIT_CODE]),
"%s failed." % testcase.testname)
def testUnsyncQueueOverflowNotificationTest(self):
"""This test operates on the unsynchronized queue.
Mirror testcase: UnsynchronizedWriteClient, OverflowNotificationTest.
For unsynchronized queue, multiple readers can recover from
a write overflow condition.
"""
# Server writes twice, overflows the queue.
asserts.assertTrue(
self._tests_msgq.requestWriteFmqUnsync(self.MAX_NUM_MSG),
"Server should write successfully.")
asserts.assertTrue(
self._tests_msgq.requestWriteFmqUnsync(self.MAX_NUM_MSG),
"Server should write successfully even if queue overflows.")
# Both clients fail to read.
read_data1 = []
read_data2 = []
asserts.assertFalse(
self._unsync_client1.read(read_data1, self.MAX_NUM_MSG),
"Client 1 should fail to read because queue overflows.")
asserts.assertFalse(
self._unsync_client2.read(read_data2, self.MAX_NUM_MSG),
"Client 2 should fail to read because queue overflows.")
# Server writes the data again.
asserts.assertTrue(
self._tests_msgq.requestWriteFmqUnsync(self.MAX_NUM_MSG),
"Server should write successfully.")
# Both clients should be able to read.
asserts.assertTrue(
self._unsync_client1.read(read_data1, self.MAX_NUM_MSG),
"Client 1 should read successfully.")
asserts.assertTrue(
self._unsync_client2.read(read_data2, self.MAX_NUM_MSG),
"Client 2 should read successfully.")
def testUnsyncQueueLargeInputTest3(self):
"""This test operates on the unsynchronized queue.
Mirrors testcase: UnsynchronizedWriteClient, LargeInputTest3.
Client writes until the queue is full, and writes one more,
which overflows the queue. Read should fail after that.
Client writes again, and server should be able to read again.
"""
write_data = generateSequentialData(self.MAX_NUM_MSG)
# Client fills up the queue.
asserts.assertTrue(
self._unsync_client1.write(write_data, self.MAX_NUM_MSG),
"Client should write successfully.")
asserts.assertEqual(0, self._unsync_client1.availableToWrite())
# Client attempts to write one more, still succeeds.
asserts.assertTrue(
self._unsync_client1.write([1], 1),
"Client should write successfully " +
"even if queue is full for unsynchronized queue.")
# Server fails to read because queue overflows.
asserts.assertFalse(
self._tests_msgq.requestReadFmqUnsync(self.MAX_NUM_MSG),
"Server should fail to read because queue overflows.")
# Do another interaction, and both should succeed.
asserts.assertTrue(
self._unsync_client1.write(write_data, self.MAX_NUM_MSG),
"Client should write successfully.")
asserts.assertTrue(
self._tests_msgq.requestReadFmqUnsync(self.MAX_NUM_MSG),
"Server should read successfully.")
def testRun(self):
"""Runs test in audio Loopback.apk and process the test results."""
latencies = []
confidences = []
for i in range(0, self.ITERATION):
self.dut.shell.Execute(["rm -f %s/*" % self.FULL_DATA_DIR_PATH])
self.dut.shell.Execute([
"am start -n org.drrickorang.loopback/.LoopbackActivity "
"--es FileName %s/%s --ei AudioLevel 11 --ei TestType 222;" %
(self.TEST_DATA_DIR, self.TEST_FILE_PREFIX)
])
# wait until the test finished.
results = self.dut.shell.Execute(["ls %s" % self.TEST_FILE_NAME])
while results[const.EXIT_CODE][0]:
logging.info("Test is running...")
sleep(1)
results = self.dut.shell.Execute(
["ls %s" % self.TEST_FILE_NAME])
results = self.dut.shell.Execute(["cat %s" % self.TEST_FILE_NAME])
asserts.assertFalse(results[const.EXIT_CODE][0],
"Fail to get the test output")
stdout_lines = results[const.STDOUT][0].split("\n")
for line in stdout_lines:
if line.startswith("LatencyMs"):
latencies.append(
int(float(line.replace("LatencyMs = ", "")) * 1000000))
if line.startswith("LatencyConfidence"):
confidences.append(
float(line.replace("LatencyConfidence = ", "")))
self.ProcessTestResults(latencies, confidences)
def _VerifyBatchResult(self, gtest_result):
'''Check a gtest test case result in batch mode
Args:
gtest_result: GtestTestCase object, representing gtest result
'''
asserts.assertFalse(gtest_result.failure_message,
gtest_result.failure_message)
def testOemHook(self):
"""Test to ensure the deprecated IOemHook HAL is not present"""
logging.debug("Check if API is present in VINTF")
try:
asserts.assertFalse(self.apiNameInVintf("IOemHook"),
"IOemHook cannot be present")
except TypeError as e:
asserts.fail(str(e))
def testNoSystemMountPoint(self):
"""Checks there is no /system mount point."""
# The format of /proc/mounts is:
# <partition> <mount point> <file system> <mount options> ...
results = self._shell.Execute("cat /proc/mounts | cut -d\" \" -f2")
mount_points = results[const.STDOUT][0].split()
logging.info('Mount points on the device: %s', mount_points)
asserts.assertFalse("/system" in mount_points,
"/system mount point shouldn't exist")
def testUnsyncQueueReadWhenEmpty(self):
"""This test operates on the unsynchronized queue.
Mirrors testcase: UnsynchronizedWriteClient, ReadWhenEmpty.
Read should fail when queue is empty.
"""
asserts.assertEqual(self._unsync_client1.availableToRead(), 0)
asserts.assertFalse(
self._unsync_client1.read([], 2),
"Client should fail to read because queue is empty.")
def VerifyTestResult(self, test_case, command_results):
'''Parse Gtest xml result output.
Sample
<testsuites tests="1" failures="1" disabled="0" errors="0"
timestamp="2017-05-24T18:32:10" time="0.012" name="AllTests">
<testsuite name="ConsumerIrHidlTest"
tests="1" failures="1" disabled="0" errors="0" time="0.01">
<testcase name="TransmitTest" status="run" time="0.01"
classname="ConsumerIrHidlTest">
<failure message="hardware/interfaces..." type="">
<![CDATA[hardware/interfaces...]]>
</failure>
</testcase>
</testsuite>
</testsuites>
Args:
test_case: GtestTestCase object, the test being run. This param
is not currently used in this method.
command_results: dict of lists, shell command result
'''
asserts.assertTrue(command_results, 'Empty command response.')
asserts.assertEqual(len(command_results), 3,
'Abnormal command response.')
for item in command_results.values():
asserts.assertEqual(
len(item), 2,
'Abnormal command result length: %s' % command_results)
for stderr in command_results[const.STDERR]:
if stderr and stderr.strip():
for line in stderr.split('\n'):
logging.error(line)
xml_str = command_results[const.STDOUT][1].strip()
if self.batch_mode:
self._ParseBatchResults(test_case, xml_str)
return
for stdout in command_results[const.STDOUT]:
if stdout and stdout.strip():
for line in stdout.split('\n'):
logging.info(line)
asserts.assertFalse(
command_results[const.EXIT_CODE][1],
'Failed to show Gtest XML output: %s' % command_results)
root = xml.etree.ElementTree.fromstring(xml_str)
asserts.assertEqual(root.get('tests'), '1', 'No tests available')
if root.get('errors') != '0' or root.get('failures') != '0':
messages = [x.get('message') for x in root.findall('.//failure')]
asserts.fail('\n'.join([x for x in messages if x]))
asserts.skipIf(root.get('disabled') == '1', 'Gtest test case disabled')
def testIllegalBlocking(self):
"""Test blocking is not allowed in unsynchronized queue.
This test operates on queue 3, and tests that blocking is not allowed
in unsynchronized queue.
"""
write_data = self.GetRandomFloats(2048)
asserts.assertFalse(
self._queue3_writer.writeBlocking(write_data, 2048, 100 * 1000000),
"Blocking operation should fail in unsynchronized queue.")
def VerifyTestResult(self, test_case, command_results):
'''Parse command result.
Args:
command_results: dict of lists, shell command result
'''
asserts.assertTrue(command_results, 'Empty command response.')
asserts.assertFalse(
any(command_results[const.EXIT_CODE]),
'Test {} failed with the following results: {}'.format(
test_case, command_results))
def testWriteFull(self):
"""Test writes fail when queue is full. """
write_data = self.GetRandomIntegers(2048)
# This write should succeed.
asserts.assertTrue(
self._queue1_writer.write(write_data, 2048),
"Writer should write successfully.")
# This write should fail because queue is full.
asserts.assertFalse(
self._queue1_writer.write(write_data, 2048),
"Writer should fail because queue is full.")
def testProcSysrqTrigger(self):
filepath = "/proc/sysrq-trigger"
# This command only performs a best effort attempt to remount all
# filesystems. Check that it doesn't throw an error.
self.dut.adb.shell("\"echo u > %s\"" % filepath)
# Reboot the device.
self.dut.adb.shell("\"echo b > %s\"" % filepath)
asserts.assertFalse(self.dut.hasBooted(), "Device is still alive.")
self.dut.waitForBootCompletion()
self.dut.rootAdb()
def _GetServiceInstanceCombinations(self, trace_path):
"""Create all combinations of instances for all services recorded in
the trace file.
Args:
trace_path: string, full path of a given trace file
Returns:
A list of all service instance combinations.
"""
registered_services = []
service_instances = {}
self.shell.Execute("chmod 755 %s" % self.replayer_binary_path)
results = self.shell.Execute(
"LD_LIBRARY_PATH=%s:$LD_LIBRARY_PATH %s --list_service %s" %
(self.custom_ld_library_path, self.replayer_binary_path,
trace_path))
asserts.assertFalse(
results[const.EXIT_CODE][0],
'Failed to list test cases. EXIT_CODE: %s\n STDOUT: %s\n STDERR: %s\n'
% (results[const.EXIT_CODE][0], results[const.STDOUT][0],
results[const.STDERR][0]))
# parse the results to get the registered service list.
for line in results[const.STDOUT][0].split('\n'):
line = str(line)
if line.startswith('hal_service: '):
service = line[len('hal_service: '):]
registered_services.append(service)
for service in registered_services:
testable, service_names = hal_service_name_utils.GetHalServiceName(
self.shell, service, self.abi_bitness,
self.run_as_compliance_test)
if not testable:
self.skipAllTests("Hal: %s is not testable, "
"skip all tests." % service)
return []
if service_names:
service_instances[service] = service_names
self._test_hal_services.add(service)
else:
self.skipAllTests("No service name found for: %s, "
"skip all tests." % service)
return []
logging.info("registered service instances: %s", service_instances)
service_instance_combinations = \
hal_service_name_utils.GetServiceInstancesCombinations(
registered_services, service_instances)
return service_instance_combinations
def testHelloWorld(self):
'''A simple testcase which sends a hello world command.'''
command = 'echo Hello World!'
res1 = self.shell1.Execute(command)
res2 = self.shell2.Execute(command)
asserts.assertFalse(
any(res1[const.EXIT_CODE]),
'command for device 1 failed: %s' % res1) # checks the exit code
asserts.assertFalse(
any(res2[const.EXIT_CODE]),
'command for device 2 failed: %s' % res2) # checks the exit code
def testBlockingTimeout(self):
"""Test blocking timeout.
This test operates on queue2, and tests that reader should time out
because there is not data available.
"""
read_data = []
read_success = self._queue2_reader.readBlocking(
read_data, 5, 100 * 1000000)
asserts.assertFalse(
read_success,
"Reader blocking should time out because there is no data to read."
)
def RunBenchmark(self, bits):
"""Runs the native binary and parses its result.
Args:
bits: integer (32 or 64), the number of bits in a word chosen
at the compile time (e.g., 32- vs. 64-bit library).
"""
# Runs the benchmark.
logging.info(
"Start to run the benchmark with HIDL mode %s (%s bit mode)",
self.hidl_hal_mode, bits)
binary = "/data/local/tmp/%s/libhwbinder_benchmark%s" % (bits, bits)
results = self.dut.shell.Execute([
"chmod 755 %s" % binary,
"VTS_ROOT_PATH=/data/local/tmp " \
"LD_LIBRARY_PATH=/system/lib%s:/data/local/tmp/%s/hw:"
"/data/local/tmp/%s:$LD_LIBRARY_PATH "
"%s -m %s --benchmark_format=json" %
(bits, bits, bits, binary, self.hidl_hal_mode.encode("utf-8"))
])
# Parses the result.
asserts.assertEqual(len(results[const.STDOUT]), 2)
logging.info("stderr: %s", results[const.STDERR][1])
logging.info("stdout: %s", results[const.STDOUT][1])
asserts.assertFalse(any(results[const.EXIT_CODE]),
"HwBinderPerformanceTest failed.")
parser = benchmark_parser.GoogleBenchmarkJsonParser(
results[const.STDOUT][1])
label_result = parser.GetArguments()
value_result = parser.GetRealTime()
table_name = "hwbinder_vector_roundtrip_latency_benchmark_%sbits" % bits
self.addTableToResult(table_name, parser.ToTable())
# To upload to the web DB.
self.web.AddProfilingDataLabeledVector(
table_name,
label_result,
value_result,
x_axis_label="Message Size (Bytes)",
y_axis_label="Roundtrip HwBinder RPC Latency (naonseconds)")
# Assertions to check the performance requirements
for label, value in zip(label_result, value_result):
if label in self.THRESHOLD[bits]:
asserts.assertLess(
value, self.THRESHOLD[bits][label],
"%s ns for %s is longer than the threshold %s ns" %
(value, label, self.THRESHOLD[bits][label]))
def testUnsyncQueueLargeInputTest2(self):
"""This test operates on the unsynchronized queue.
Mirrors testcase: UnsynchronizedWriteClient, LargeInputTest2.
Server attempts to write more than the queue capacity and fails.
"""
asserts.assertEqual(0, self._unsync_client1.availableToRead())
# Server attempts to write more than the queue capacity.
asserts.assertFalse(
self._tests_msgq.requestWriteFmqUnsync(self.MAX_NUM_MSG + 1),
"Server should fail because it writes more than queue capacity.")
# Check there is still no data for client.
asserts.assertEqual(0, self._unsync_client1.availableToRead())
asserts.assertFalse(
self._unsync_client1.read([], 1),
"Client should fail to read because queue is empty.")
def testHostBinary(self):
"""Tests host-side binaries."""
android_build_top = os.getenv("ANDROID_BUILD_TOP", "")
asserts.assertTrue(
android_build_top,
"$ANDROID_BUILD_TOP is not set. Please run lunch <build target>")
for binary_test_source in self.binary_test_source:
binary_test_source = str(binary_test_source)
binary_path = os.path.join(android_build_top, binary_test_source)
cmd_result = cmd_utils.ExecuteShellCommand(binary_path)
asserts.assertFalse(
any(results[cmd_utils.EXIT_CODE]),
"Test failed with the following results:\n "
"command result: %s" % results)
def testKernelNetworking(self):
"""Android kernel unit test."""
# Push the test binary to target device.
self.shell('mkdir -p %s' %
path_utils.TargetDirName(self.target_bin_path))
self.dut.adb.push('%s %s' % (self.host_bin_path, self.target_bin_path))
self.shell('chmod 777 %s' % self.target_bin_path)
# Execute the test binary.
result = self.shell(self.target_bin_path, no_except=True)
logging.info('stdout: %s', result[const.STDOUT])
logging.error('stderr: %s', result[const.STDERR])
logging.info('exit code: %s', result[const.EXIT_CODE])
asserts.assertFalse(
result[const.EXIT_CODE],
'kernel_net_tests binary returned non-zero exit code.')
def generateNonPassFilterTests(self):
'''Generate test cases for filter non passing test names.'''
logging.info('generating tests for SHOULD_NOT_PASS_FILTER: %s',
self.SHOULD_NOT_PASS_FILTER)
self.runGeneratedTests(test_func=lambda x: asserts.assertFalse(
self.CheckPassFilter(x), 'Filter should not accept test name: %s' %
x),
settings=self.SHOULD_NOT_PASS_FILTER,
name_func=lambda x: 'filter_not_accept_%s' % x)
def testSyncQueueWriteWhenFull(self):
"""This test operates on the synchronized queue.
Mirrors testcase: SynchronizedReadWriteClient, WriteWhenFull.
Write should fail when queue is full.
"""
write_data = generateSequentialData(self.MAX_NUM_MSG)
# Client writes.
asserts.assertTrue(
self._sync_client.write(write_data, self.MAX_NUM_MSG),
"Client should write successfully.")
asserts.assertEqual(self._sync_client.availableToWrite(), 0)
# Client tries to write more, fails.
asserts.assertFalse(
self._sync_client.write([1], 1),
"Client should fail to write because queue is full.")
# Server should read data back correctly.
asserts.assertTrue(
self._tests_msgq.requestReadFmqSync(self.MAX_NUM_MSG),
"Server should read successfully")
def testSyncQueueLargeInputTest3(self):
"""This test operates on the synchronized queue.
Mirrors testcase: SynchronizedReadWriteClient, LargeInputTest3.
Client writes until the queue is full, attempts to write one more,
and fails.
"""
write_data = generateSequentialData(self.MAX_NUM_MSG)
# Client fills up the queue.
asserts.assertTrue(
self._sync_client.write(write_data, self.MAX_NUM_MSG),
"Client should write successfully.")
asserts.assertEqual(0, self._sync_client.availableToWrite())
# Client attempts to write one more, fails.
asserts.assertFalse(
self._sync_client.write([1], 1),
"Client should fail to write because queue is full.")
# Server reads back data from client.
asserts.assertTrue(
self._tests_msgq.requestReadFmqSync(self.MAX_NUM_MSG),
"Server should read successfully.")
def AssertTestResult(self, result):
"""Asserts that testcase finished as expected.
Checks that device is in responsive state. If not, waits for boot
then reports test as failure. If it is, asserts that all test commands
returned exit code 0.
Args:
result: dict(str, str, int), stdout, stderr and return code
from test run.
"""
if self._dut.hasBooted():
exit_code = result[const.EXIT_CODE]
asserts.skipIf(exit_code == config.ExitCode.POC_TEST_SKIP,
"Test case was skipped.")
asserts.assertFalse(exit_code == config.ExitCode.POC_TEST_FAIL,
"Test case failed.")
else:
self._dut.waitForBootCompletion()
self._dut.rootAdb()
self.PushFiles()
asserts.fail("Test case left the device in unresponsive state.")
