def setup_test(self): if getattr(self, "qxdm_log", True): start_qxdm_loggers(self.log, self.android_devices) ensure_phones_idle(self.log, self.android_devices) toggle_airplane_mode(self.log, self.ad, True) self.ad.adb.shell("logcat -c -b all", ignore_status=True) return True
def _send_message(self, ads): selection = random.randrange(0, 2) message_type_map = {0: "SMS", 1: "MMS"} max_length_map = {0: self.max_sms_length, 1: self.max_mms_length} min_length_map = {0: self.min_sms_length, 1: self.min_mms_length} length = random.randrange(min_length_map[selection], max_length_map[selection] + 1) text = rand_ascii_str(length) message_content_map = {0: [text], 1: [("Mms Message", text, None)]} message_func_map = { 0: sms_send_receive_verify, 1: mms_send_receive_verify } message_type = message_type_map[selection] self.result_info["Total %s" % message_type] += 1 begin_time = get_current_epoch_time() start_qxdm_loggers(self.log, self.android_devices) if not message_func_map[selection](self.log, ads[0], ads[1], message_content_map[selection]): self.log.error("%s of length %s from %s to %s fails", message_type, length, ads[0].serial, ads[1].serial) self.result_info["%s failure" % message_type] += 1 if message_type == "SMS" or self.result_info["%s failure" % message_type] == 1: self._take_bug_report("%s_%s_failure" % (self.test_name, message_type), begin_time) return False else: self.log.info("%s of length %s from %s to %s succeed", message_type_map[selection], length, ads[0].serial, ads[1].serial) return True
def _mobile_data_toggling(self, setup="volte"): # ModePref change to non-LTE begin_time = get_device_epoch_time(self.dut) start_qxdm_loggers(self.log, self.android_devices) result = True self.result_info["Data Toggling Request Total"] += 1 test_name = "%s_data_toggling_iter_%s" % ( self.test_name, self.result_info["Data Toggling Request Total"]) log_msg = "[Test Case] %s" % test_name self.log.info("%s begin", log_msg) self.dut.droid.logI("%s begin" % log_msg) self.dut.adb.shell("svc data disable") time.sleep(WAIT_TIME_AFTER_MODE_CHANGE) self.dut.adb.shell("svc data enable") if not self._check_data(): result = False elif setup == "volte" and not phone_idle_volte(self.log, self.dut): result = False self.dut.droid.logI("%s end" % log_msg) self.dut.log.info("%s end", log_msg) if not result: self.result_info["Data Toggling Failure"] += 1 try: self._ad_take_extra_logs(self.dut, test_name, begin_time) self._ad_take_bugreport(self.dut, test_name, begin_time) except Exception as e: self.log.exception(e) return False else: self.result_info["Data Toggling Success"] += 1 return True
def _data_call_test(self, sub_id, generation): self.dut.log.info(dict(self.result_info)) begin_time = get_device_epoch_time(self.dut) start_qxdm_loggers(self.log, self.android_devices) self.result_info["Network Change Request Total"] += 1 test_name = "%s_network_change_test_iter_%s" % ( self.test_name, self.result_info["Network Change Request Total"]) log_msg = "[Test Case] %s" % test_name self.log.info("%s begin", log_msg) self.dut.droid.logI("%s begin" % log_msg) if not ensure_network_generation_for_subscription( self.log, self.dut, sub_id, generation) or not self._check_data(): self.result_info["Network Change Failure"] += 1 self.dut.droid.logI("%s end" % log_msg) self.dut.log.info("%s end", log_msg) try: self._ad_take_extra_logs(self.dut, test_name, begin_time) self._ad_take_bugreport(self.dut, test_name, begin_time) except Exception as e: self.log.warning(e) return False if not self._mobile_data_toggling(setup=None): return False return True
def setup_test(self): if getattr(self, "qxdm_log", True): start_qxdm_loggers(self.log, self.android_devices) ensure_phones_idle(self.log, self.android_devices) toggle_airplane_mode(self.log, self.ad, True) self.ad.adb.shell("setprop net.lte.ims.volte.provisioned 1", ignore_status=True) return True
def _test_stress_cbrs(self, mo_mt): """ Test CBRS/SSIM VoLTE Stress mo_mt: indicating this call sequence is MO or MT. Valid input: DIRECTION_MOBILE_ORIGINATED and DIRECTION_MOBILE_TERMINATED. Returns: True if pass; False if fail. """ if (mo_mt not in [ DIRECTION_MOBILE_ORIGINATED, DIRECTION_MOBILE_TERMINATED ]): self.log.error("Invalid parameters.") return False ads = [self.android_devices[0], self.android_devices[1]] total_iteration = self.stress_test_number fail_count = collections.defaultdict(int) self.cbrs_subid, self.default_subid = get_cbrs_and_default_sub_id( ads[0]) self.log.info("Total iteration = %d.", total_iteration) current_iteration = 1 for i in range(1, total_iteration + 1): msg = "Stress Call Test Iteration: <%s> / <%s>" % (i, total_iteration) begin_time = get_current_epoch_time() self.log.info(msg) start_qxdm_loggers(self.log, self.android_devices, begin_time) iteration_result = self._cbrs_call_sequence( ads, mo_mt, self._phone_setup_volte, self._is_current_data_on_cbrs, self._test_data_browsing_success_using_sl4a, self._is_phone_in_call_volte, self._is_current_data_on_default, self._test_data_browsing_success_using_sl4a, self._test_data_browsing_success_using_sl4a, self._is_current_data_on_cbrs, True) self.log.info("Result: %s", iteration_result) if iteration_result: self.log.info(">----Iteration : %d/%d succeed.----<", i, total_iteration) else: fail_count["cbrs_fail"] += 1 self.log.error(">----Iteration : %d/%d failed.----<", i, total_iteration) self._take_bug_report("%s_IterNo_%s" % (self.test_name, i), begin_time) current_iteration += 1 test_result = True for failure, count in fail_count.items(): if count: self.log.error("%s: %s %s failures in %s iterations", self.test_name, count, failure, total_iteration) test_result = False return test_result
def setup_test(self): try: if getattr(self, "qxdm_log", True): start_qxdm_loggers(self.log, self.android_devices) self.ad.droid.telephonyFactoryReset() except Exception as e: self.ad.log.error(e) toggle_airplane_mode_by_adb(self.log, self.ad, True) self.ad.adb.shell("setprop net.lte.ims.volte.provisioned 1", ignore_status=True) # get a handle to virtual phone self.virtualPhoneHandle = self.anritsu.get_VirtualPhone() return True
def setup_test(self): try: if getattr(self, "qxdm_log", True): start_qxdm_loggers(self.log, self.android_devices) self.ad.droid.telephonyFactoryReset() if self.ad.sim_card == "VzW12349": self.ad.droid.imsSetVolteProvisioning(True) except Exception as e: self.ad.log.error(e) toggle_airplane_mode_by_adb(self.log, self.ad, True) # get a handle to virtual phone self.virtualPhoneHandle = self.anritsu.get_VirtualPhone() return True
def setup_test(self): if getattr(self, "qxdm_log", True): start_qxdm_loggers(self.log, self.android_devices) ensure_phone_default_state(self.log, self.ad, check_subscription=False) toggle_airplane_mode_by_adb(self.log, self.ad, True) try: if self.ad.sim_card == "VzW12349": self.ad.droid.imsSetVolteProvisioning(True) except Exception as e: self.ad.log.error(e) # get a handle to virtual phone self.virtualPhoneHandle = self.anritsu.get_VirtualPhone() return True
def _data_download(self, file_names=["5MB", "10MB", "20MB", "50MB"]): begin_time = get_current_epoch_time() start_qxdm_loggers(self.log, self.android_devices) self.dut.log.info(dict(self.result_info)) selection = random.randrange(0, len(file_names)) file_name = file_names[selection] self.result_info["Internet Connection Check Total"] += 1 if not self.internet_connection_check_method(self.log, self.dut): rat = self.dut.adb.getprop("gsm.network.type") self.dut.log.info("Network in RAT %s", rat) if self.dut_incall and not is_rat_svd_capable(rat.upper()): self.result_info[ "Expected Incall Internet Connection Check Failure"] += 1 return True else: self.result_info["Internet Connection Check Failure"] += 1 test_name = "%s_internet_connection_No_%s_failure" % ( self.test_name, self.result_info["Internet Connection Check Failure"]) try: self._ad_take_extra_logs(self.dut, test_name, begin_time) self._ad_take_bugreport(self.dut, test_name, begin_time) except Exception as e: self.log.exception(e) return False else: self.result_info["Internet Connection Check Success"] += 1 self.result_info["File Download Total"] += 1 if not active_file_download_test( self.log, self.dut, file_name, method=self.file_download_method): self.result_info["File Download Failure"] += 1 if self.result_info["File Download Failure"] == 1: try: self._ad_take_extra_logs( self.dut, "%s_file_download_failure" % self.test_name, begin_time) self._ad_take_bugreport( self.dut, "%s_file_download_failure" % self.test_name, begin_time) except Exception as e: self.log.exception(e) return False else: self.result_info["File Download Success"] += 1 return True
def _make_phone_call(self, ads): self.result_info["Total Calls"] += 1 begin_time = get_current_epoch_time() start_qxdm_loggers(self.log, self.android_devices) if not call_setup_teardown( self.log, ads[0], ads[1], ad_hangup=ads[random.randrange(0, 2)], wait_time_in_call=random.randrange( self.min_phone_call_duration, self.max_phone_call_duration)): self.log.error("Call setup and teardown failed.") self.result_info["Call Failure"] += 1 self._take_bug_report("%s_call_failure" % self.test_name, begin_time) return False self.log.info("Call setup and teardown succeed.") return True
def data_test(self): failure = 0 total_count = 0 #file_names = ["5MB", "10MB", "20MB", "50MB", "200MB", "512MB", "1GB"] #wifi download is very slow in lab, limit the file size upto 200MB file_names = ["5MB", "10MB", "20MB", "50MB", "200MB"] while time.time() < self.finishing_time: total_count += 1 begin_time = get_current_epoch_time() start_qxdm_loggers(self.log, self.android_devices) try: self.dut.log.info(dict(self.result_info)) self.result_info["Total file download"] += 1 selection = random.randrange(0, len(file_names)) file_name = file_names[selection] if not active_file_download_test(self.log, self.dut, file_name): self.result_info["File download failure"] += 1 failure += 1 if self.result_info["File download failure"] == 1: self._take_bug_report( "%s_file_download_failure" % self.test_name, begin_time) self.dut.droid.goToSleepNow() time.sleep(random.randrange(0, self.max_sleep_time)) except IGNORE_EXCEPTIONS as e: self.log.error("Exception error %s", str(e)) self.result_info["Exception Errors"] += 1 if self.result_info["Exception Errors"] > EXCEPTION_TOLERANCE: self.log.error("Too many exception error %s", IGNORE_EXCEPTIONS) return False except Exception as e: self.log.error(e) return False self.dut.log.info("File download test failure: %s/%s", failure, total_count) if failure / total_count > 0.1: return False else: return True
def setup_test(self): if getattr(self, "qxdm_log", True): if not self.user_params.get("qxdm_log_mask_cfg", None): if "wfc" in self.test_name: for ad in self.android_devices: if not getattr(ad, "qxdm_logger_command", None) or ( "IMS_DS_CNE_LnX_Golden.cfg" not in getattr( ad, "qxdm_logger_command", "")): set_qxdm_logger_command( ad, "IMS_DS_CNE_LnX_Golden.cfg") else: for ad in self.android_devices: if not getattr(ad, "qxdm_logger_command", None) or ( "IMS_DS_CNE_LnX_Golden.cfg" in getattr( ad, "qxdm_logger_command", "")): set_qxdm_logger_command(ad, None) start_qxdm_loggers(self.log, self.android_devices, self.begin_time) if getattr(self, "sdm_log", False): start_sdm_loggers(self.log, self.android_devices) if getattr(self, "tcpdump_log", False) or "wfc" in self.test_name: mask = getattr(self, "tcpdump_mask", "all") interface = getattr(self, "tcpdump_interface", "wlan0") start_tcpdumps( self.android_devices, begin_time=self.begin_time, interface=interface, mask=mask) else: stop_tcpdumps(self.android_devices) for ad in self.android_devices: if self.skip_reset_between_cases: ensure_phone_idle(self.log, ad) else: ensure_phone_default_state(self.log, ad) for session in ad._sl4a_manager.sessions.values(): ed = session.get_event_dispatcher() ed.clear_all_events() output = ad.adb.logcat("-t 1") match = re.search(r"\d+-\d+\s\d+:\d+:\d+.\d+", output) if match: ad.test_log_begin_time = match.group(0)
def _prefnetwork_mode_change(self, sub_id): # ModePref change to non-LTE begin_time = get_device_epoch_time(self.dut) start_qxdm_loggers(self.log, self.android_devices) self.result_info["Network Change Request Total"] += 1 test_name = "%s_network_change_iter_%s" % ( self.test_name, self.result_info["Network Change Request Total"]) log_msg = "[Test Case] %s" % test_name self.log.info("%s begin", log_msg) self.dut.droid.logI("%s begin" % log_msg) network_preference_list = [ NETWORK_MODE_TDSCDMA_GSM_WCDMA, NETWORK_MODE_WCDMA_ONLY, NETWORK_MODE_GLOBAL, NETWORK_MODE_CDMA, NETWORK_MODE_GSM_ONLY ] network_preference = random.choice(network_preference_list) set_preferred_network_mode_pref(self.log, self.dut, sub_id, network_preference) time.sleep(WAIT_TIME_AFTER_MODE_CHANGE) self.dut.log.info("Current Voice RAT is %s", get_current_voice_rat(self.log, self.dut)) # ModePref change back to with LTE if not phone_setup_volte(self.log, self.dut): self.dut.log.error("Phone failed to enable VoLTE.") self.result_info["VoLTE Setup Failure"] += 1 self.dut.droid.logI("%s end" % log_msg) self.dut.log.info("%s end", log_msg) try: self._ad_take_extra_logs(self.dut, test_name, begin_time) self._ad_take_bugreport(self.dut, test_name, begin_time) except Exception as e: self.log.exception(e) return False else: self.result_info["VoLTE Setup Success"] += 1 return True
def setup_test(self): if getattr(self, "qxdm_log", True): start_qxdm_loggers(self.log, self.android_devices) ensure_phones_idle(self.log, self.android_devices) toggle_airplane_mode(self.log, self.ad, True) return True
def _send_message(self, max_wait_time=2 * MAX_WAIT_TIME_SMS_RECEIVE): if self.single_phone_test: ads = [self.dut, self.dut] else: ads = self.android_devices[:] random.shuffle(ads) selection = random.randrange(0, 2) message_type_map = {0: "SMS", 1: "MMS"} max_length_map = {0: self.max_sms_length, 1: self.max_mms_length} min_length_map = {0: self.min_sms_length, 1: self.min_mms_length} length = random.randrange(min_length_map[selection], max_length_map[selection] + 1) message_func_map = { 0: sms_send_receive_verify, 1: mms_send_receive_verify } rat = self.dut.adb.getprop("gsm.network.type") self.dut.log.info("Network in RAT %s", rat) if self.dut_incall and not is_rat_svd_capable(rat.upper()): self.dut.log.info("In call data not supported, test SMS only") selection = 0 message_type = message_type_map[selection] the_number = self.result_info["%s Total" % message_type] + 1 begin_time = get_device_epoch_time(self.dut) test_name = "%s_No_%s_%s" % (self.test_name, the_number, message_type) start_qxdm_loggers(self.log, self.android_devices) log_msg = "[Test Case] %s" % test_name self.log.info("%s begin", log_msg) for ad in self.android_devices: if self.user_params.get("turn_on_tcpdump", True): start_adb_tcpdump(ad, interface="any", mask="all") if not getattr(ad, "messaging_droid", None): ad.messaging_droid, ad.messaging_ed = ad.get_droid() ad.messaging_ed.start() else: try: if not ad.messaging_droid.is_live: ad.messaging_droid, ad.messaging_ed = ad.get_droid() ad.messaging_ed.start() else: ad.messaging_ed.clear_all_events() except Exception: ad.log.info("Create new sl4a session for messaging") ad.messaging_droid, ad.messaging_ed = ad.get_droid() ad.messaging_ed.start() ad.messaging_droid.logI("%s begin" % log_msg) text = "%s: " % test_name text_length = len(text) if length < text_length: text = text[:length] else: text += rand_ascii_str(length - text_length) message_content_map = {0: [text], 1: [(test_name, text, None)]} result = message_func_map[selection](self.log, ads[0], ads[1], message_content_map[selection], max_wait_time) self.log.info("%s end", log_msg) for ad in self.android_devices: ad.messaging_droid.logI("%s end" % log_msg) if not result: self.result_info["%s Total" % message_type] += 1 if message_type == "SMS": self.log.error("%s fails", log_msg) self.result_info["%s Failure" % message_type] += 1 else: rat = self.dut.adb.getprop("gsm.network.type") self.dut.log.info("Network in RAT %s", rat) if self.dut_incall and not is_rat_svd_capable(rat.upper()): self.dut.log.info( "In call data not supported, MMS failure expected") self.result_info["Expected In-call MMS failure"] += 1 return True else: self.log.error("%s fails", log_msg) self.result_info["MMS Failure"] += 1 try: self._take_bug_report(test_name, begin_time) except Exception as e: self.log.exception(e) return False else: self.result_info["%s Total" % message_type] += 1 self.log.info("%s succeed", log_msg) self.result_info["%s Success" % message_type] += 1 return True
def stress_test(self, setup_func=None, network_check_func=None, test_sms=False, test_video=False): for ad in self.android_devices: #check for sim and service ensure_phone_subscription(self.log, ad) if setup_func and not setup_func(): self.log.error("Test setup %s failed", setup_func.__name__) return False fail_count = collections.defaultdict(int) for i in range(1, self.phone_call_iteration + 1): msg = "Stress Call Test %s Iteration: <%s> / <%s>" % ( self.test_name, i, self.phone_call_iteration) begin_time = get_current_epoch_time() self.log.info(msg) iteration_result = True ensure_phones_idle(self.log, self.android_devices) if not self._setup_phone_call(test_video, phone_call_duration=self.phone_call_duration): fail_count["dialing"] += 1 iteration_result = False self.log.error("%s call dialing failure.", msg) else: if network_check_func and not network_check_func( self.log, self.caller): fail_count["caller_network_check"] += 1 last_call_drop_reason(self.caller, begin_time) iteration_result = False self.log.error("%s network check %s failure.", msg, network_check_func.__name__) if network_check_func and not network_check_func( self.log, self.callee): fail_count["callee_network_check"] += 1 last_call_drop_reason(self.callee, begin_time) iteration_result = False self.log.error("%s network check failure.", msg) if not verify_incall_state(self.log, [self.caller, self.callee], True): self.log.error("%s call dropped.", msg) iteration_result = False fail_count["drop"] += 1 self._hangup_call() if test_sms and not sms_send_receive_verify( self.log, self.caller, self.callee, [rand_ascii_str(180)]): fail_count["sms"] += 1 self.log.info("%s %s", msg, iteration_result) if not iteration_result: self._take_bug_report("%s_CallNo_%s" % (self.test_name, i), begin_time) start_qxdm_loggers(self.log, self.android_devices) if self.sleep_time_between_test_iterations: self.caller.droid.goToSleepNow() self.callee.droid.goToSleepNow() time.sleep(self.sleep_time_between_test_iterations) test_result = True for failure, count in fail_count.items(): if count: self.log.error("%s: %s %s failures in %s iterations", self.test_name, count, failure, self.phone_call_iteration) test_result = False return test_result
def _make_phone_call(self, call_verification_func=None): ads = self.android_devices[:] if not self.single_phone_test: random.shuffle(ads) the_number = self.result_info["Call Total"] + 1 duration = random.randrange(self.min_phone_call_duration, self.max_phone_call_duration) result = True test_name = "%s_No_%s_phone_call" % (self.test_name, the_number) log_msg = "[Test Case] %s" % test_name self.log.info("%s for %s seconds begin", log_msg, duration) begin_time = get_device_epoch_time(ads[0]) for ad in self.android_devices: if self.user_params.get("turn_on_tcpdump", True): start_adb_tcpdump(ad, interface="any", mask="all") if not getattr(ad, "droid", None): ad.droid, ad.ed = ad.get_droid() ad.ed.start() else: try: if not ad.droid.is_live: ad.droid, ad.ed = ad.get_droid() ad.ed.start() else: ad.ed.clear_all_events() except Exception: ad.log.info("Create new sl4a session for phone call") ad.droid, ad.ed = ad.get_droid() ad.ed.start() ad.droid.logI("%s begin" % log_msg) start_qxdm_loggers(self.log, self.android_devices, begin_time) failure_reasons = set() self.dut_incall = True if self.single_phone_test: call_setup_result = initiate_call( self.log, self.dut, self.call_server_number, incall_ui_display=INCALL_UI_DISPLAY_BACKGROUND ) and wait_for_in_call_active(self.dut, 60, 3) else: call_setup_result = call_setup_teardown( self.log, ads[0], ads[1], ad_hangup=None, verify_caller_func=call_verification_func, verify_callee_func=call_verification_func, wait_time_in_call=0, incall_ui_display=INCALL_UI_DISPLAY_BACKGROUND) if not call_setup_result: call_logs = ads[0].search_logcat( "ActivityManager: START u0 {act=android.intent.action.CALL", begin_time) messaging_logs = ads[0].search_logcat( "com.google.android.apps.messaging/.ui.conversation.ConversationActivity", begin_time) if call_logs and messaging_logs: if messaging_logs[-1]["datetime_obj"] - call_logs[-1]["datetime_obj"] < 5: ads[0].log.info( "Call setup failure due to simultaneous activities") self.result_info[ "Call Setup Failure With Simultaneous Activity"] += 1 return True self.log.error("%s: Setup Call failed.", log_msg) failure_reasons.add("Setup") result = False else: elapsed_time = 0 check_interval = 5 while (elapsed_time < duration): check_interval = min(check_interval, duration - elapsed_time) time.sleep(check_interval) elapsed_time += check_interval time_message = "at <%s>/<%s> second." % (elapsed_time, duration) for ad in ads: if not call_verification_func(self.log, ad): ad.log.warning("Call is NOT in correct %s state at %s", call_verification_func.__name__, time_message) if call_verification_func.__name__ == "is_phone_in_call_iwlan": if is_phone_in_call(self.log, ad): if getattr(ad, "data_rat_state_error_count", 0) < 1: setattr(ad, "data_rat_state_error_count", 1) continue failure_reasons.add("Maintenance") last_call_drop_reason(ad, begin_time) hangup_call(self.log, ads[0]) result = False else: ad.log.info("Call is in correct %s state at %s", call_verification_func.__name__, time_message) if not result: break if not hangup_call(self.log, ads[0]): failure_reasons.add("Teardown") result = False for ad in ads: if not wait_for_call_id_clearing(ad, []) or ad.droid.telecomIsInCall(): ad.log.error("Fail to hang up call") failure_reasons.add("Teardown") result = False self.result_info["Call Total"] += 1 for ad in self.android_devices: try: ad.droid.logI("%s end" % log_msg) except: pass self.log.info("%s end", log_msg) self.dut_incall = False if not result: self.log.info("%s failed", log_msg) if self.gps_log_file: gps_info = job.run( "tail %s" % self.gps_log_file, ignore_status=True) if gps_info.stdout: gps_log_path = os.path.join(self.log_path, test_name, "gps_logs.txt") utils.create_dir(gps_log_path) job.run( "tail %s > %s" % (self.gps_log_file, gps_log_path), ignore_status=True) self.log.info("gps log:\n%s", gps_info.stdout) else: self.log.warning("Fail to get gps log %s", self.user_params["gps_log_file"]) for reason in failure_reasons: self.result_info["Call %s Failure" % reason] += 1 for ad in ads: log_path = os.path.join(self.log_path, test_name, "%s_binder_logs" % ad.serial) utils.create_dir(log_path) ad.pull_files(BINDER_LOGS, log_path) try: self._take_bug_report(test_name, begin_time) except Exception as e: self.log.exception(e) for ad in ads: if ad.droid.telecomIsInCall(): hangup_call_by_adb(ad) else: self.log.info("%s test succeed", log_msg) self.result_info["Call Success"] += 1 if self.result_info["Call Total"] % 50 == 0: for ad in ads: synchronize_device_time(ad) if not check_is_wifi_connected(self.log, ad, self.wifi_network_ssid): ensure_wifi_connected(self.log, ad, self.wifi_network_ssid, self.wifi_network_pass) force_connectivity_metrics_upload(ad) time.sleep(300) wifi_toggle_state(self.log, ad, False) if self.get_binder_logs: log_path = os.path.join(self.log_path, "%s_binder_logs" % test_name, "%s_binder_logs" % ad.serial) utils.create_dir(log_path) ad.pull_files(BINDER_LOGS, log_path) return result