def test_pull(self, device: Device, mp_tmp_dir):
storage = DeviceStorage(device)
local_path = os.path.join(mp_tmp_dir, "somefile")
remote_path = "/".join([storage.external_storage_location, "touchedfile"])
device.execute_remote_cmd("shell", "touch", remote_path)
storage.pull(remote_path=remote_path, local_path=local_path)
assert os.path.exists(local_path)
def execute_test_plan(self,
test_application: TestApplication,
test_plan: Iterator[TestSuite],
test_listener: TestListener,
global_uploadables: List[Tuple[str, str]] = None):
"""
Execute a test plan (a collection of test suites)
:param test_application: test application to run
:param test_plan: iterator with each element being a tuple of test suite name and list of string arguments
to provide to an executionof "adb instrument". The test suit name is
used to report start and end of each test suite via the test_listener
:param test_listener: used to report test results as they occur
:raises: asyncio.TimeoutError if test or test suite times out based on this orchestrator's configuration
"""
loop = asyncio.get_event_loop()
device_restoration = self._DeviceRestoration(test_application.device)
device_storage = DeviceStorage(test_application.device)
self._test_butler_service = ServiceApplication.from_apk(
self._test_butler_apk_path, test_application.device)
# add testbutler tag for processing
if self._test_butler_service:
line_parser: LineParser = TestButlerCommandParser(
self._test_butler_service,
app_under_test=test_application,
listener=device_restoration)
self.add_logcat_monitor("TestButler", line_parser, priority='I')
if self._tag_monitors:
log.debug("Creating logcat monitoring task")
loop.create_task(self._process_logcat_tags(
test_application.device))
# ADD USER-DEFINED TASKS
async def timer():
"""
Timer to timeout if future is not presented in given timeout for overall test suite execution
"""
for local_path, remote_path in global_uploadables or []:
device_storage.push(local_path=local_path,
remote_path=remote_path)
await asyncio.wait_for(
self._execute_plan(test_plan=test_plan,
test_application=test_application,
test_listener=test_listener,
device_restoration=device_restoration),
self._instrumentation_timeout)
try:
loop.run_until_complete(timer(
)) # execute plan until completed or until timeout is reached
finally:
for _, remote_path in global_uploadables or []:
device_storage.remove(remote_path, recursive=True)
def test_push_remove(self, device: Device):
storage = DeviceStorage(device)
remote_location = "/".join(
[storage.external_storage_location, "some_file"])
with suppress(Exception):
storage.remove(remote_location)
output = device.execute_remote_cmd("shell",
"ls",
device.external_storage_location,
capture_stdout=True)
if os.path.basename(remote_location) in output:
raise Exception("Error: did not expect file %s on remote device" %
remote_location)
storage.push(local_path=(os.path.abspath(__file__)),
remote_path=remote_location)
output = device.execute_remote_cmd("shell",
"ls",
device.external_storage_location,
capture_stdout=True)
assert os.path.basename(remote_location) in output
storage.remove(remote_location)
output = device.execute_remote_cmd("shell",
"ls",
device.external_storage_location,
capture_stdout=True)
assert not os.path.basename(remote_location) in output
def test_make_dir(self, device: Device):
storage = DeviceStorage(device)
new_remote_dir = "/".join(
[storage.external_storage_location, "a", "b", "c", "d"])
# assure dir does not already exist:
with suppress(Exception):
storage.remove(new_remote_dir, recursive=True)
try:
output = device.execute_remote_cmd("shell",
"ls",
"-d",
new_remote_dir,
capture_stdout=True)
# expect "no such directory" error leading to exception, but just in case:
assert new_remote_dir not in output
except Device.CommandExecutionFailureException as e:
assert "no such" in str(e).lower()
storage.make_dir(new_remote_dir)
output = device.execute_remote_cmd("shell",
"ls",
"-d",
new_remote_dir,
capture_stdout=True)
assert new_remote_dir in output
async def _execute_plan(
self, test_plan: Iterator[TestSuite],
test_application: TestApplication, test_listener: TestListener,
device_restoration: "AndroidTestOrchestrator._DeviceRestoration"):
"""
Loop over items in test plan and execute one by one, restoring device settings and properties on each
iteration.
:param test_plan: generator of tuples of (test_suite_name, list_of_instrument_arguments)
:param test_application: test application containing (remote) runner to execute tests
:param device_restoration: to restore device on each iteration
"""
# clear logcat for fresh start
try:
DeviceLog(test_application.device).clear()
except Device.CommandExecutionFailureException as e:
# sometimes logcat -c will give and error "failed to clear 'main' log'; Ugh
log.error("clearing logcat failed: " + str(e))
async def single_buffered_test_suite():
"""
This coroutine buffers one item out of the test plan to be immediately available
This is to accommodate the case where iteration over each item in the plan could take a
little bit of processing time (for example, being asynchronously pulled from a file). It is
expected that this shouldn't take much time (10s of ms), but over 1000s of tests this allows
that overhead of getting the next test to be absorbed while a test is executing on the remote
device
"""
test_suite = next(test_plan, (None, None))
for next_item in test_plan:
yield test_suite
test_suite = next_item
yield test_suite
instrumentation_parser = InstrumentationOutputParser(test_listener)
# add timer that times timeout if any INDIVIDUAL test takes too long
if self._test_timeout is not None:
instrumentation_parser.add_test_execution_listener(
Timer(self._test_timeout))
# TASK-3: capture logcat to file and markers for beginning/end of each test
device_log = DeviceLog(test_application.device)
device_storage = DeviceStorage(test_application.device)
with device_log.capture_to_file(output_path=os.path.join(
self._artifact_dir, "logcat.txt")) as log_capture:
# log_capture is to listen to test status to mark beginning/end of each test run:
instrumentation_parser.add_test_execution_listener(log_capture)
async for test_suite in single_buffered_test_suite():
test_listener.test_suite_started(test_suite.name)
try:
for local_path, remote_path in test_suite.uploadables:
device_storage.push(local_path=local_path,
remote_path=remote_path)
async with await test_application.run(*test_suite.arguments
) as lines:
async for line in lines:
instrumentation_parser.parse_line(line)
except Exception as e:
print(trace(e))
test_listener.test_suite_errored(test_suite.name, 1,
str(e))
finally:
test_listener.test_suite_ended(
test_suite.name,
instrumentation_parser.total_test_count,
instrumentation_parser.execution_time)
device_restoration.restore()
for _, remote_path in test_suite.uploadables:
try:
device_storage.remove(remote_path, recursive=True)
except Exception as e:
log.error(
"Failed to remove temporary test vector %s from device"
% remote_path)
# capture logcat markers (begin/end of each test/test suite)
marker_output_path = os.path.join(self._artifact_dir,
'log_markers.txt')
if os.path.exists(marker_output_path):
os.remove(marker_output_path)
with open(marker_output_path, 'w') as f:
for marker, pos in log_capture.markers.items():
f.write("%s=%s\n" % (marker, str(pos)))
def test_external_storage_location(self, device: Device):
assert DeviceStorage(device).external_storage_location.startswith("/")
def test_pull_invalid_remote_path(self, device: Device, mp_tmp_dir):
storage = DeviceStorage(device)
local = os.path.join(str(mp_tmp_dir), "nosuchfile")
with pytest.raises(Exception):
storage.pull(remote_path="/no/such/file", local_path=local)
assert not os.path.exists(local)
def test_push_invalid_remote_path(self, device: Device):
storage = DeviceStorage(device)
remote_location = "/a/bogus/remote/location"
with pytest.raises(Exception):
storage.push(local_path=(os.path.abspath(__file__)),
remote_path=remote_location)
def test_external_storage_location(self, device: Device):
assert DeviceStorage(device).external_storage_location == "/sdcard"
def test_pull(self, device: Device, tmpdir):
storage = DeviceStorage(device)
local = os.path.join(tmpdir, "somefile")
storage.pull(remote_path="/etc/init/bootstat.rc", local_path=local)
assert os.path.exists(local)