class GdDeviceBase(ABC): """ Base GD device class that covers common traits which assumes that the device must be driven by a driver-like backing process that takes following command line arguments: --grpc-port: main entry port for facade services --root-server-port: management port for starting and stopping services --btsnoop: path to btsnoop HCI log --signal-port: signaling port to indicate that backing process is started --rootcanal-port: root-canal HCI port, optional """ WAIT_CHANNEL_READY_TIMEOUT_SECONDS = 10 def __init__(self, grpc_port: str, grpc_root_server_port: str, signal_port: str, cmd: List[str], label: str, type_identifier: str, name: str, verbose_mode: bool): """Base GD device, common traits for both device based and host only GD cert tests :param grpc_port: main gRPC service port :param grpc_root_server_port: gRPC root server port :param signal_port: signaling port for backing process start up :param cmd: list of arguments to run in backing process :param label: device label used in logs :param type_identifier: device type identifier used in logs :param name: name of device used in logs """ # Must be at the first line of __init__ method values = locals() arguments = [values[arg] for arg in inspect.getfullargspec(GdDeviceBase.__init__).args if arg != "verbose_mode"] asserts.assert_true(all(arguments), "All arguments to GdDeviceBase must not be None nor empty") asserts.assert_true(all(cmd), "cmd list should not have None nor empty component") self.verbose_mode = verbose_mode self.grpc_root_server_port = int(grpc_root_server_port) self.grpc_port = int(grpc_port) self.signal_port = int(signal_port) self.name = name self.type_identifier = type_identifier self.label = label # logging.log_path only exists when this is used in an ACTS test run. self.log_path_base = get_current_context().get_full_output_path() self.test_runner_base_path = \ get_current_context().get_base_output_path() self.backing_process_log_path = os.path.join(self.log_path_base, '%s_%s_backing_logs.txt' % (self.type_identifier, self.label)) if "--btsnoop=" not in " ".join(cmd): cmd.append("--btsnoop=%s" % os.path.join(self.log_path_base, '%s_btsnoop_hci.log' % self.label)) if "--btsnooz=" not in " ".join(cmd): cmd.append("--btsnooz=%s" % os.path.join(self.log_path_base, '%s_btsnooz_hci.log' % self.label)) if "--btconfig=" not in " ".join(cmd): cmd.append("--btconfig=%s" % os.path.join(self.log_path_base, '%s_bt_config.conf' % self.label)) self.cmd = cmd self.environment = os.environ.copy() if "cert" in self.label: self.terminal_color = TerminalColor.BLUE else: self.terminal_color = TerminalColor.YELLOW def setup(self): """Set up this device for test, must run before using this device - After calling this, teardown() must be called when test finishes - Should be executed after children classes' setup() methods :return: """ # Ensure signal port is available # signal port is the only port that always listen on the host machine asserts.assert_true( make_ports_available([self.signal_port]), "[%s] Failed to make signal port available" % self.label) # Start backing process with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as signal_socket: # Setup signaling socket signal_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) signal_socket.bind(("localhost", self.signal_port)) signal_socket.listen(1) signal_socket.settimeout(300) # 5 minute timeout for blocking socket operations # Start backing process logging.debug("Running %s" % " ".join(self.cmd)) self.backing_process = subprocess.Popen( self.cmd, cwd=get_gd_root(), env=self.environment, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) asserts.assert_true(self.backing_process, msg="Cannot start backing_process at " + " ".join(self.cmd)) asserts.assert_true( is_subprocess_alive(self.backing_process), msg="backing_process stopped immediately after running " + " ".join(self.cmd)) # Wait for process to be ready logging.debug("Waiting for backing_process accept.") signal_socket.accept() self.backing_process_logger = AsyncSubprocessLogger( self.backing_process, [self.backing_process_log_path], log_to_stdout=self.verbose_mode, tag=self.label, color=self.terminal_color) # Setup gRPC management channels self.grpc_root_server_channel = grpc.insecure_channel("localhost:%d" % self.grpc_root_server_port) self.grpc_channel = grpc.insecure_channel("localhost:%d" % self.grpc_port) if self.verbose_mode: self.grpc_channel = grpc.intercept_channel(self.grpc_channel, LoggingClientInterceptor(self.label)) # Establish services from facades self.rootservice = facade_rootservice_pb2_grpc.RootFacadeStub(self.grpc_root_server_channel) self.hal = hal_facade_pb2_grpc.HciHalFacadeStub(self.grpc_channel) self.controller_read_only_property = facade_rootservice_pb2_grpc.ReadOnlyPropertyStub(self.grpc_channel) self.hci = hci_facade_pb2_grpc.HciFacadeStub(self.grpc_channel) self.l2cap = l2cap_facade_pb2_grpc.L2capClassicModuleFacadeStub(self.grpc_channel) self.l2cap_le = l2cap_le_facade_pb2_grpc.L2capLeModuleFacadeStub(self.grpc_channel) self.iso = iso_facade_pb2_grpc.IsoModuleFacadeStub(self.grpc_channel) self.hci_acl_manager = acl_manager_facade_pb2_grpc.AclManagerFacadeStub(self.grpc_channel) self.hci_le_acl_manager = le_acl_manager_facade_pb2_grpc.LeAclManagerFacadeStub(self.grpc_channel) self.hci_le_initiator_address = le_initiator_address_facade_pb2_grpc.LeInitiatorAddressFacadeStub( self.grpc_channel) self.hci_controller = controller_facade_pb2_grpc.ControllerFacadeStub(self.grpc_channel) self.hci_controller.GetMacAddressSimple = lambda: self.hci_controller.GetMacAddress(empty_proto.Empty()).address self.hci_controller.GetLocalNameSimple = lambda: self.hci_controller.GetLocalName(empty_proto.Empty()).name self.hci_le_advertising_manager = le_advertising_manager_facade_pb2_grpc.LeAdvertisingManagerFacadeStub( self.grpc_channel) self.hci_le_scanning_manager = le_scanning_manager_facade_pb2_grpc.LeScanningManagerFacadeStub( self.grpc_channel) self.neighbor = neighbor_facade_pb2_grpc.NeighborFacadeStub(self.grpc_channel) self.security = security_facade_pb2_grpc.SecurityModuleFacadeStub(self.grpc_channel) self.shim = shim_facade_pb2_grpc.ShimFacadeStub(self.grpc_channel) def get_crash_snippet_and_log_tail(self): if is_subprocess_alive(self.backing_process): return None, None return read_crash_snippet_and_log_tail(self.backing_process_log_path) def teardown(self): """Tear down this device and clean up any resources. - Must be called after setup() - Should be executed before children classes' teardown() :return: """ self.grpc_channel.close() self.grpc_root_server_channel.close() stop_signal = signal.SIGINT self.backing_process.send_signal(stop_signal) try: return_code = self.backing_process.wait(timeout=self.WAIT_CHANNEL_READY_TIMEOUT_SECONDS) except subprocess.TimeoutExpired: logging.error("[%s] Failed to interrupt backing process via SIGINT, sending SIGKILL" % self.label) stop_signal = signal.SIGKILL self.backing_process.kill() try: return_code = self.backing_process.wait(timeout=self.WAIT_CHANNEL_READY_TIMEOUT_SECONDS) except subprocess.TimeoutExpired: logging.error("Failed to kill backing process") return_code = -65536 if return_code not in [-stop_signal, 0]: logging.error("backing process %s stopped with code: %d" % (self.label, return_code)) self.backing_process_logger.stop() def wait_channel_ready(self): future = grpc.channel_ready_future(self.grpc_channel) try: future.result(timeout=self.WAIT_CHANNEL_READY_TIMEOUT_SECONDS) except grpc.FutureTimeoutError: asserts.fail("[%s] wait channel ready timeout" % self.label)
class GdAndroidDevice(GdDeviceBase): """Real Android device where the backing process is running on it """ WAIT_FOR_DEVICE_TIMEOUT_SECONDS = 180 def __init__(self, grpc_port: str, grpc_root_server_port: str, signal_port: str, cmd: List[str], label: str, type_identifier: str, name: str, serial_number: str, verbose_mode: bool): super().__init__(grpc_port, grpc_root_server_port, signal_port, cmd, label, type_identifier, name, verbose_mode) asserts.assert_true(serial_number, "serial_number must not be None nor empty") self.serial_number = serial_number self.adb = AdbProxy(serial_number) def setup(self): logging.info("Setting up device %s %s" % (self.label, self.serial_number)) asserts.assert_true(self.adb.ensure_root(), "device %s cannot run as root", self.serial_number) # Try freeing ports and ignore results self.cleanup_port_forwarding() self.sync_device_time() # Set up port forwarding or reverse or die self.tcp_forward_or_die(self.grpc_port, self.grpc_port) self.tcp_forward_or_die(self.grpc_root_server_port, self.grpc_root_server_port) self.tcp_reverse_or_die(self.signal_port, self.signal_port) # Push test binaries self.ensure_verity_disabled() self.push_or_die(os.path.join(get_gd_root(), "target", "bluetooth_stack_with_facade"), "system/bin") self.push_or_die(os.path.join(get_gd_root(), "target", "libbluetooth_gd.so"), "system/lib64") self.push_or_die(os.path.join(get_gd_root(), "target", "libgrpc++_unsecure.so"), "system/lib64") try: self.adb.shell("rm /data/misc/bluetooth/logs/btsnoop_hci.log") except AdbCommandError as error: logging.warning("Failed to remove old btsnoop log: " + str(error)) try: self.adb.shell("rm /data/misc/bluetooth/logs/btsnooz_hci.log") except AdbCommandError as error: logging.warning("Failed to remove old btsnooz log: " + str(error)) try: self.adb.shell("rm /data/misc/bluedroid/bt_config.conf") except AdbCommandError as error: logging.warning("Failed to remove old bt config: " + str(error)) try: self.adb.shell("rm /data/misc/bluedroid/bt_config.bak") except AdbCommandError as error: logging.warning("Failed to remove back up config: " + str(error)) self.ensure_no_output(self.adb.shell("svc bluetooth disable")) # Start logcat logging self.logcat_output_path = os.path.join( self.log_path_base, '%s_%s_%s_logcat_logs.txt' % (self.type_identifier, self.label, self.serial_number)) self.logcat_cmd = ["adb", "-s", self.serial_number, "logcat", "-T", "1", "-v", "year", "-v", "uid"] logging.debug("Running %s", " ".join(self.logcat_cmd)) self.logcat_process = subprocess.Popen( self.logcat_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) asserts.assert_true(self.logcat_process, msg="Cannot start logcat_process at " + " ".join(self.logcat_cmd)) asserts.assert_true( is_subprocess_alive(self.logcat_process), msg="logcat_process stopped immediately after running " + " ".join(self.logcat_cmd)) self.logcat_logger = AsyncSubprocessLogger( self.logcat_process, [self.logcat_output_path], log_to_stdout=self.verbose_mode, tag="%s_%s" % (self.label, self.serial_number), color=self.terminal_color) # Done run parent setup logging.info("Done preparation for %s, starting backing process" % self.serial_number) super().setup() def teardown(self): super().teardown() stop_signal = signal.SIGINT self.logcat_process.send_signal(stop_signal) try: return_code = self.logcat_process.wait(timeout=self.WAIT_CHANNEL_READY_TIMEOUT_SECONDS) except subprocess.TimeoutExpired: logging.error("[%s_%s] Failed to interrupt logcat process via SIGINT, sending SIGKILL" % (self.label, self.serial_number)) stop_signal = signal.SIGKILL self.logcat_process.kill() try: return_code = self.logcat_process.wait(timeout=self.WAIT_CHANNEL_READY_TIMEOUT_SECONDS) except subprocess.TimeoutExpired: logging.error("Failed to kill logcat_process %s %s" % (self.label, self.serial_number)) return_code = -65536 if return_code not in [-stop_signal, 0]: logging.error("logcat_process %s_%s stopped with code: %d" % (self.label, self.serial_number, return_code)) self.logcat_logger.stop() self.cleanup_port_forwarding() self.pull_logs(self.log_path_base) def pull_logs(self, base_dir): try: self.adb.pull("/data/misc/bluetooth/logs/btsnoop_hci.log %s" % os.path.join( base_dir, "%s_btsnoop_hci.log" % self.label)) self.adb.pull( "/data/misc/bluedroid/bt_config.conf %s" % os.path.join(base_dir, "%s_bt_config.conf" % self.label)) self.adb.pull( "/data/misc/bluedroid/bt_config.bak %s" % os.path.join(base_dir, "%s_bt_config.bak" % self.label)) except AdbCommandError as error: logging.warning("Failed to pull logs from device: " + str(error)) def cleanup_port_forwarding(self): try: self.adb.remove_tcp_forward(self.grpc_port) except AdbError as error: logging.warning("Failed to cleanup gRPC port: " + str(error)) try: self.adb.remove_tcp_forward(self.grpc_root_server_port) except AdbError as error: logging.warning("Failed to cleanup gRPC server port: " + str(error)) try: self.adb.reverse("--remove tcp:%d" % self.signal_port) except AdbError as error: logging.warning("Failed to cleanup signal port: " + str(error)) @staticmethod def ensure_no_output(result): """ Ensure a command has not output """ asserts.assert_true( result is None or len(result) == 0, msg="command returned something when it shouldn't: %s" % result) def sync_device_time(self): self.adb.shell("settings put global auto_time 0") self.adb.shell("settings put global auto_time_zone 0") device_tz = self.adb.shell("date +%z") asserts.assert_true(device_tz, "date +%z must return device timezone, " "but returned {} instead".format(device_tz)) host_tz = time.strftime("%z") if device_tz != host_tz: target_timezone = utils.get_timezone_olson_id() logging.debug("Device timezone %s does not match host timezone %s, " "syncing them by setting timezone to %s" % (device_tz, host_tz, target_timezone)) self.adb.shell("setprop persist.sys.timezone %s" % target_timezone) self.reboot() device_tz = self.adb.shell("date +%z") asserts.assert_equal( host_tz, device_tz, "Device timezone %s still does not match host " "timezone %s after reset" % (device_tz, host_tz)) self.adb.shell("date %s" % time.strftime("%m%d%H%M%Y.%S")) datetime_format = "%Y-%m-%dT%H:%M:%S%z" try: device_time = datetime.strptime(self.adb.shell("date +'%s'" % datetime_format), datetime_format) except ValueError: asserts.fail("Failed to get time after sync") return # Include ADB delay that might be longer in SSH environment max_delta_seconds = 3 host_time = datetime.now(tz=device_time.tzinfo) asserts.assert_almost_equal( (device_time - host_time).total_seconds(), 0, msg="Device time %s and host time %s off by >%dms after sync" % (device_time.isoformat(), host_time.isoformat(), int(max_delta_seconds * 1000)), delta=max_delta_seconds) def push_or_die(self, src_file_path, dst_file_path, push_timeout=300): """Pushes a file to the Android device Args: src_file_path: The path to the file to install. dst_file_path: The destination of the file. push_timeout: How long to wait for the push to finish in seconds """ out = self.adb.push('%s %s' % (src_file_path, dst_file_path), timeout=push_timeout) if 'error' in out: asserts.fail('Unable to push file %s to %s due to %s' % (src_file_path, dst_file_path, out)) def tcp_forward_or_die(self, host_port, device_port, num_retry=1): """ Forward a TCP port from host to device or fail :param host_port: host port, int, 0 for adb to assign one :param device_port: device port, int :param num_retry: number of times to reboot and retry this before dying :return: host port int """ error_or_port = self.adb.tcp_forward(host_port, device_port) if not error_or_port: logging.debug("host port %d was already forwarded" % host_port) return host_port if not isinstance(error_or_port, int): if num_retry > 0: # If requested, reboot an retry num_retry -= 1 logging.warning( "[%s] Failed to TCP forward host port %d to " "device port %d, num_retries left is %d" % (self.label, host_port, device_port, num_retry)) self.reboot() return self.tcp_forward_or_die(host_port, device_port, num_retry=num_retry) asserts.fail( 'Unable to forward host port %d to device port %d, error %s' % (host_port, device_port, error_or_port)) return error_or_port def tcp_reverse_or_die(self, device_port, host_port, num_retry=1): """ Forward a TCP port from device to host or fail :param device_port: device port, int, 0 for adb to assign one :param host_port: host port, int :param num_retry: number of times to reboot and retry this before dying :return: device port int """ error_or_port = self.adb.reverse("tcp:%d tcp:%d" % (device_port, host_port)) if not error_or_port: logging.debug("device port %d was already reversed" % device_port) return device_port try: error_or_port = int(error_or_port) except ValueError: if num_retry > 0: # If requested, reboot an retry num_retry -= 1 logging.warning( "[%s] Failed to TCP reverse device port %d to " "host port %d, num_retries left is %d" % (self.label, device_port, host_port, num_retry)) self.reboot() return self.tcp_reverse_or_die(device_port, host_port, num_retry=num_retry) asserts.fail( 'Unable to reverse device port %d to host port %d, error %s' % (device_port, host_port, error_or_port)) return error_or_port def ensure_verity_disabled(self): """Ensures that verity is enabled. If verity is not enabled, this call will reboot the phone. Note that this only works on debuggable builds. """ logging.debug("Disabling verity and remount for %s", self.serial_number) # The below properties will only exist if verity has been enabled. system_verity = self.adb.getprop('partition.system.verified') vendor_verity = self.adb.getprop('partition.vendor.verified') if system_verity or vendor_verity: self.adb.disable_verity() self.reboot() self.adb.remount() self.adb.wait_for_device(timeout=self.WAIT_FOR_DEVICE_TIMEOUT_SECONDS) def reboot(self, timeout_minutes=15.0): """Reboots the device. Reboot the device, wait for device to complete booting. """ logging.debug("Rebooting %s", self.serial_number) self.adb.reboot() timeout_start = time.time() timeout = timeout_minutes * 60 # Android sometimes return early after `adb reboot` is called. This # means subsequent calls may make it to the device before the reboot # goes through, return false positives for getprops such as # sys.boot_completed. while time.time() < timeout_start + timeout: try: self.adb.get_state() time.sleep(.1) except AdbError: # get_state will raise an error if the device is not found. We # want the device to be missing to prove the device has kicked # off the reboot. break minutes_left = timeout_minutes - (time.time() - timeout_start) / 60.0 self.wait_for_boot_completion(timeout_minutes=minutes_left) asserts.assert_true(self.adb.ensure_root(), "device %s cannot run as root after reboot", self.serial_number) def wait_for_boot_completion(self, timeout_minutes=15.0): """ Waits for Android framework to broadcast ACTION_BOOT_COMPLETED. :param timeout_minutes: number of minutes to wait """ timeout_start = time.time() timeout = timeout_minutes * 60 self.adb.wait_for_device(timeout=self.WAIT_FOR_DEVICE_TIMEOUT_SECONDS) while time.time() < timeout_start + timeout: try: completed = self.adb.getprop("sys.boot_completed") if completed == '1': return except AdbError: # adb shell calls may fail during certain period of booting # process, which is normal. Ignoring these errors. pass time.sleep(5) asserts.fail(msg='Device %s booting process timed out.' % self.serial_number)
class GdBaseTestClass(BaseTestClass): SUBPROCESS_WAIT_TIMEOUT_SECONDS = 10 def setup_class(self, dut_module, cert_module): self.dut_module = dut_module self.cert_module = cert_module self.log_path_base = get_current_context().get_full_output_path() self.verbose_mode = bool(self.user_params.get('verbose_mode', False)) for config in self.controller_configs[CONTROLLER_CONFIG_NAME]: config['verbose_mode'] = self.verbose_mode # Start root-canal if needed self.rootcanal_running = False if 'rootcanal' in self.controller_configs: self.rootcanal_running = True # Get root canal binary rootcanal = os.path.join(get_gd_root(), "root-canal") asserts.assert_true(os.path.isfile(rootcanal), "Root canal does not exist at %s" % rootcanal) # Get root canal log self.rootcanal_logpath = os.path.join(self.log_path_base, 'rootcanal_logs.txt') # Make sure ports are available rootcanal_config = self.controller_configs['rootcanal'] rootcanal_test_port = int(rootcanal_config.get( "test_port", "6401")) rootcanal_hci_port = int(rootcanal_config.get("hci_port", "6402")) rootcanal_link_layer_port = int( rootcanal_config.get("link_layer_port", "6403")) asserts.assert_true( make_ports_available((rootcanal_test_port, rootcanal_hci_port, rootcanal_link_layer_port)), "Failed to make root canal ports available") # Start root canal process rootcanal_cmd = [ rootcanal, str(rootcanal_test_port), str(rootcanal_hci_port), str(rootcanal_link_layer_port) ] self.log.debug("Running %s" % " ".join(rootcanal_cmd)) self.rootcanal_process = subprocess.Popen(rootcanal_cmd, cwd=get_gd_root(), env=os.environ.copy(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, universal_newlines=True) asserts.assert_true(self.rootcanal_process, msg="Cannot start root-canal at " + str(rootcanal)) asserts.assert_true( is_subprocess_alive(self.rootcanal_process), msg="root-canal stopped immediately after running") self.rootcanal_logger = AsyncSubprocessLogger( self.rootcanal_process, [self.rootcanal_logpath], log_to_stdout=self.verbose_mode, tag="rootcanal", color=TerminalColor.MAGENTA) # Modify the device config to include the correct root-canal port for gd_device_config in self.controller_configs.get("GdDevice"): gd_device_config["rootcanal_port"] = str(rootcanal_hci_port) # Parse and construct GD device objects self.register_controller(importlib.import_module('cert.gd_device'), builtin=True) self.dut = self.gd_devices[1] self.cert = self.gd_devices[0] def teardown_class(self): if self.rootcanal_running: stop_signal = signal.SIGINT self.rootcanal_process.send_signal(stop_signal) try: return_code = self.rootcanal_process.wait( timeout=self.SUBPROCESS_WAIT_TIMEOUT_SECONDS) except subprocess.TimeoutExpired: logging.error( "Failed to interrupt root canal via SIGINT, sending SIGKILL" ) stop_signal = signal.SIGKILL self.rootcanal_process.kill() try: return_code = self.rootcanal_process.wait( timeout=self.SUBPROCESS_WAIT_TIMEOUT_SECONDS) except subprocess.TimeoutExpired: logging.error("Failed to kill root canal") return_code = -65536 if return_code != 0 and return_code != -stop_signal: logging.error("rootcanal stopped with code: %d" % return_code) self.rootcanal_logger.stop() def setup_test(self): self.dut.rootservice.StartStack( facade_rootservice.StartStackRequest( module_under_test=facade_rootservice.BluetoothModule.Value( self.dut_module), )) self.cert.rootservice.StartStack( facade_rootservice.StartStackRequest( module_under_test=facade_rootservice.BluetoothModule.Value( self.cert_module), )) self.dut.wait_channel_ready() self.cert.wait_channel_ready() def teardown_test(self): self.cert.rootservice.StopStack(facade_rootservice.StopStackRequest()) self.dut.rootservice.StopStack(facade_rootservice.StopStackRequest()) def __getattribute__(self, name): attr = super().__getattribute__(name) if not callable(attr) or not GdBaseTestClass.__is_entry_function(name): return attr @wraps(attr) def __wrapped(*args, **kwargs): try: return attr(*args, **kwargs) except RpcError as e: exception_info = "".join( traceback.format_exception(e.__class__, e, e.__traceback__)) raise signals.TestFailure( "RpcError during test\n\nRpcError:\n\n%s\n%s" % (exception_info, self.__dump_crashes())) return __wrapped __ENTRY_METHODS = { "setup_class", "teardown_class", "setup_test", "teardown_test" } @staticmethod def __is_entry_function(name): return name.startswith( "test_") or name in GdBaseTestClass.__ENTRY_METHODS def __dump_crashes(self): """ :return: formatted stack traces if found, or last few lines of log """ dut_crash, dut_log_tail = self.dut.get_crash_snippet_and_log_tail() cert_crash, cert_log_tail = self.cert.get_crash_snippet_and_log_tail() rootcanal_crash = None rootcanal_log_tail = None if self.rootcanal_running and not is_subprocess_alive( self.rootcanal_process): rootcanal_crash, roocanal_log_tail = read_crash_snippet_and_log_tail( self.rootcanal_logpath) crash_detail = "" if dut_crash or cert_crash or rootcanal_crash: if rootcanal_crash: crash_detail += "rootcanal crashed:\n\n%s\n\n" % rootcanal_crash if dut_crash: crash_detail += "dut stack crashed:\n\n%s\n\n" % dut_crash if cert_crash: crash_detail += "cert stack crashed:\n\n%s\n\n" % cert_crash else: if rootcanal_log_tail: crash_detail += "rootcanal log tail:\n\n%s\n\n" % rootcanal_log_tail if dut_log_tail: crash_detail += "dut log tail:\n\n%s\n\n" % dut_log_tail if cert_log_tail: crash_detail += "cert log tail:\n\n%s\n\n" % cert_log_tail return crash_detail