def get_tc_run_command(self, binary_name, suite_name, tc_name,
package_name, output_file):
command = '{}{}/runner.sh {} {} {} {}'.format(
self.get_platform_shell_prefix(), self.app_path, binary_name,
suite_name, tc_name, output_file)
print_runner_output(command)
return command
def print_tc_list(self, file_path, dynamic_runner):
fp = open(file_path, "w")
list_of_testcase = dynamic_runner.get_tc_list()
for testcase in list_of_testcase:
command = 'binary_name:{}, suite_name:{}, tc_name:{}, package_name:{}'.format(testcase.binary_name, testcase.suite_name, testcase.tc_name, testcase.package_name)
fp.write (command + '\n')
fp.close()
print_runner_output('TC List Generated!!')
def get_devices(self):
command = "sdb devices | awk '/[a-zA-Z0-9]+[ \t]+device[ \t]+[a-zA-Z0-9-]+$/{print $1}'"
sdb_devices = subprocess.check_output(
command, shell=True).decode('ascii').strip().splitlines()
print_runner_output(sdb_devices)
if len(sdb_devices) > 0:
return sdb_devices
return []
def set_gtest_tc_list(self, test_result_dir, given_module, file_filter,
given_testsuites, given_testcases, dynamic_runner):
binary_list = dynamic_runner.get_binary_list()
sz = 0
for binary_name in binary_list:
print_runner_output(binary_name)
if file_filter:
if file_filter not in binary_name:
continue
if given_module:
if len(binary_name.split('_')) < 3:
continue
if binary_name.split('_')[1] not in given_module:
continue
testsuite = ''
command = dynamic_runner.get_tc_list_command(binary_name)
rc = subprocess.check_output(command, shell=True)
log = re.sub(r'(b\'|\')', '', str(rc))
log = dynamic_runner.remove_invalid_character_from_log(log)
for line in log.split('\n'):
line = line.strip()
if not line:
continue
if line.endswith('.') == True:
testsuite = line
continue
if given_testsuites and testsuite[:-1] not in given_testsuites:
continue
if given_testcases and line not in given_testcases:
continue
if testsuite != '' and line != '':
if dynamic_runner.tc_scenario:
if testsuite.split(
'_')[-1] not in dynamic_runner.tc_scenario:
continue
dynamic_runner.add_new_tc(binary_name, testsuite[:-1],
line, '')
def check_connectivity(self):
command = "{} ip addr show wlan0 | grep 'inet ' | cut -d' ' -f6 | cut -d/ -f1".format(
self.get_platform_shell_prefix())
android_ip = subprocess.check_output(
command, shell=True).decode('ascii').strip()
command = "ping -c 3 {} | grep 'packet loss'| cut -d ',' -f 3 | cut -d '%' -f 1".format(
android_ip)
packet_loss = subprocess.check_output(
command, shell=True).decode('ascii').strip()
print_runner_output('packet_loss : {}%'.format(packet_loss))
if len(packet_loss) > 0 and int(packet_loss) < 50:
return True
return False
def check_connectivity(self):
command = "{} shell ifconfig wlan0 | grep 'inet ' | cut -d ':' -f 2| cut -d ' ' -f 1".format(
self.get_platform_shell_prefix())
tizen_ip = subprocess.check_output(command,
shell=True).decode('ascii').strip()
command = "ping -c 3 {} | grep 'packet loss'| cut -d ',' -f 3 | cut -d '%' -f 1".format(
tizen_ip)
packet_loss = subprocess.check_output(
command, shell=True).decode('ascii').strip()
print_runner_output("packet_loss : {}%".format(packet_loss))
if len(packet_loss) > 0 and int(packet_loss) < 50:
return True
return False
def get_binary_list(self):
command = '{}ls {}/{}*{}'.format(self.get_platform_shell_prefix(),
self.app_path, TC_BIN_PREFIX,
TC_BIN_SUFFIX)
print_runner_output(command)
process = subprocess.Popen([command],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
log, err = process.communicate(timeout=30)
log = str(log)
log = self.remove_invalid_character_from_log(log)
log = log.replace(self.app_path + os.sep, '')
log = log.replace(self.app_path, '')
log = log.replace('\n', ' ').strip()
log = re.sub(' +', ' ', log)
return log.split(' ')
file_filter = opts.file_filter
given_module = opts.module
package_name = opts.package_name
test_result_dir = opts.test_result_dir
tc_verdict_file_path = opts.verdict_path
save_verdict = opts.save_verdict
TestRunOption.max_total_count = int(opts.max_total_count)
TestRunOption.min_pass_count = int(opts.min_pass_count)
TestRunOption.max_timeout_count = int(opts.max_timeout_count)
if save_verdict:
set_file_handler(given_module, platform_type, tc_verdict_file_path)
#sys.stdout = Multi_Logger(given_module, platform_type, tc_verdict_file_path)
print_runner_output('All parameters have taken')
dynamic_runner = None
if 'linux' == platform_type:
dynamic_runner = LinuxTestRunner()
elif 'tizen' == platform_type:
dynamic_runner = TizenTestRunner()
elif 'android' == platform_type:
tc_framework = opts.tc_framework
if not tc_framework:
tc_framework = TESTFW_TYPES.JUNIT
if TESTFW_TYPES.GTEST == tc_framework:
dynamic_runner = AndroidGtestRunner()
elif TESTFW_TYPES.JUNIT == tc_framework:
dynamic_runner = AndroidJunitRunner()
def set_junit_tc_list(self, test_result_dir, given_module,
given_package_name, given_testsuites,
given_testcases, dynamic_runner):
for testcase_type in TESTCASE_TYPES:
print_runner_output('testcase_type: {}'.format(testcase_type))
if dynamic_runner.tc_scenario:
found = False
for temp in dynamic_runner.tc_scenario:
if temp.lower() == testcase_type.lower():
found = True
break
if not found:
continue
for module in TESTSUITE_MODULES:
if given_testsuites:
found = False
for given_suite in given_testsuites:
if module in given_suite:
found = True
break
if not found:
continue
if given_module:
if module.lower() not in given_module.lower():
continue
if given_package_name:
package_name = given_package_name
else:
package_name = 'org.iotivity.test.' + module.lower(
) + '.tc.' + testcase_type.lower()
print_runner_output('package_name: {}'.format(package_name))
cwd = os.getcwd()
print_runner_output(cwd)
build_dir = os.path.join(cwd, self.test_root, 'bin',
dynamic_runner.platform_type,
module.lower(), 'intermediates',
'classes', 'debug',
package_name.replace('.', os.sep))
print_runner_output(build_dir)
if not os.path.exists(build_dir):
print_runner_output(
'path {} is not exist'.format(build_dir))
continue
os.chdir(build_dir)
print_runner_output(os.getcwd())
file_list = [
f for f in os.listdir(build_dir)
if os.path.isfile(os.path.join(build_dir, f))
]
for suite in file_list:
if "$" not in suite and suite.endswith('Test.class'):
suite_name = suite.split('.', 1)[0]
if given_testsuites and suite_name not in given_testsuites:
continue
class_command = "find -name '{}.class' | xargs javap -p".format(
suite_name)
rc = subprocess.check_output(class_command, shell=True)
log = re.sub(r'(b\'|\')', '', str(rc))
for line in log.split('\\n'):
line = re.sub(r'(b\'|\')', '', str(line.strip()))
if 'public void test' in line:
begin_index = line.find('test')
end_index = line.find('(')
tc_name = line[begin_index:end_index]
if given_testcases and tc_name not in given_testcases:
continue
dynamic_runner.add_new_tc(
'', suite_name, tc_name, package_name)
os.chdir(cwd)
def run_selected_testcase(self, binary_name, suite_name, tc_name, package_name, dynamic_runner, tc_utility_action):
fw_output_file = 'temporary_test_output'
platform_type = dynamic_runner.platform_type
target = dynamic_runner.target
build_type = dynamic_runner.build_type
transport = dynamic_runner.transport
network = dynamic_runner.network
module = dynamic_runner.module
search_text, expected, search_file_type = dynamic_runner.get_pass_criteria()
crash_texts = dynamic_runner.get_crash_text_list()
command = dynamic_runner.get_tc_run_command(binary_name, suite_name, tc_name, package_name, fw_output_file)
print_runner_output(command)
if os.path.exists(fw_output_file):
os.remove(fw_output_file)
dynamic_runner.remove_fw_output_file(fw_output_file)
failure_msg_list = []
for attempt in range(0, 2):
log = ''
result = 0
delta = 0
status = GT_ATT_STATUS[GT_LOG.OK]
is_execution_finshed = False
failure_msg_list = []
dynamic_runner.clear_device_log()
print_runner_output('current_path: {}'.format(os.getcwd()))
dynamic_runner.change_current_dir_to_app_path()
#function_name = '{}_{}_tc_pre_action'.format(module, platform_type)
#if hasattr(TcUtilityAction, function_name) and callable(getattr(TcUtilityAction, function_name)):
# getattr(tc_utility_action, function_name)()
#else:
# print_runner_output('no {} function found'.format(function_name))
start_time = datetime.now()
try:
def start_tc_execution():
nonlocal log
nonlocal is_execution_finshed
rc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True, bufsize=1)
#encoding utf-8 wasn't working, ISO-8859-1 was worked for a testcase
for line in io.TextIOWrapper(rc.stdout, encoding="ISO-8859-1"):
print_to_file(line.strip())
log += line
for crash_text in crash_texts:
if crash_text in log:
break
is_execution_finshed = True
t = threading.Thread(target=start_tc_execution)
t.start()
cnt = 0
timeout = self.timeout_seconds / 10
while t.is_alive() and cnt < timeout:
print_runner_output('Observe Session No: {}'.format(cnt))
t.join(10)
cnt += 1
if is_execution_finshed:
break
for crash_text in crash_texts:
if crash_text in log:
failure_msg_list.append("crashed")
status = GT_ATT_STATUS[GT_LOG.CRASHED]
result = 3
break
if result == 0 and not is_execution_finshed:
failure_msg_list.append("hang")
status = GT_ATT_STATUS[GT_LOG.HANG]
result = 2
if result == 0:
text_found = False
if search_file_type == TestRunOption.XML_PASS_CRITERIA:
dynamic_runner.download_fw_output_file(fw_output_file)
if os.path.exists(fw_output_file):
contents = open(fw_output_file, 'r').read()
text_found = True
else:
print_runner_output('No output file found')
else:
contents = log
text_found = True
if text_found and ((expected and contents.find(search_text) == -1) or (not expected and contents.find(search_text) >= 0)):
failure_msg_list.append(dynamic_runner.get_failure_msg(log))
result = 1
if not text_found:
failure_msg_list.append("unexpected")
result = 4
except subprocess.CalledProcessError:
show_exeception()
failure_msg_list.append("crashed")
result = 3
status = GT_ATT_STATUS[GT_LOG.CRASHED]
end_time = datetime.now()
delta = end_time - start_time
#function_name = '{}_{}_tc_post_action'.format(module, platform_type)
#if hasattr(TcUtilityAction, function_name) and callable(getattr(TcUtilityAction, function_name)):
# getattr(tc_utility_action, function_name)(log_file)
#else:
# print_runner_output('no {} function found'.format(function_name))
print_runner_output('executing {} complete'.format(command))
print_runner_output('current_path {}'.format(os.getcwd()))
dynamic_runner.change_back_to_runner_path()
if result == 0:
break
if dynamic_runner.check_connectivity():
break
else:
failure_msg_list.append("low network quality")
xml_output = '<testsuites>\n'
xml_output += '\t<testsuite name="{}">\n'.format(suite_name)
xml_output += '\t\t<testcase name="{}" status="{}" time="{}">\n'.format(tc_name, status, int(delta.total_seconds() * 1000))
for failure_msg in failure_msg_list:
xml_output += ('\t\t\t<failure message="{}"> </failure>\n'.format(failure_msg))
xml_output += '\t\t</testcase>\n'
xml_output += '\t</testsuite>\n'
xml_output += '</testsuites>\n'
timestring = datetime.now().strftime("%Y%m%d_%H%M%S.%f")
process_id = binary_name
if not process_id:
process_id = package_name
file_name = "{}_{}_{}_{}_{}_{}_{}" .format(platform_type, target, build_type, transport, network, timestring, process_id.replace('.', '_'))
log_file_path = os.path.join(self.test_result_dir, file_name + '.log')
xml_file_path = os.path.join(self.test_result_dir, file_name + '.xml')
if os.path.isfile(log_file_path) or os.path.isfile(xml_file_path):
print_runner_output('', Colors.FAIL, 'File name exist !!!\nRunner Internal Error', Colors.ENDC)
return command, 4
dynamic_runner.append_device_log(log_file_path)
if not os.path.exists(self.test_result_dir):
print_runner_output('{} path not found'.format(self.test_result_dir))
result_path_parent = self.test_result_dir
result_path_count = 0
while result_path_parent:
pos = result_path_parent.rfind(os.sep)
if pos == -1:
break
result_path_parent = result_path_parent[:pos]
if os.path.exists(result_path_parent):
print_runner_output('{} path found'.format(result_path_parent))
file_list = os.listdir(result_path_parent)
print_runner_output('printing file list ...')
for f in file_list:
print_runner_output(f)
print_runner_output('file list printed')
break
else:
print_runner_output('{} path not found'.format(result_path_parent))
result_path_count += 1
if result_path_count > 20:
break
return command, 4
if not os.path.isdir(self.test_result_dir):
print_runner_output('{} directory not found'.format(self.test_result_dir))
return command, 4
if not os.path.isfile(log_file_path):
print_runner_output('No log file {} found'.format(log_file_path))
log_file = open(log_file_path, 'a')
log_file.write('\n' + log)
xml_file = open(xml_file_path, 'w')
xml_file.write(xml_output)
return command, result
def run_selected_testcases(self, dynamic_runner, verdict_file_path, save_verdict):
verdict_types = ['pass', 'fail', 'timeout', 'crash', 'unexpected']
color_types = [Colors.OKGREEN, Colors.FAIL, Colors.FAIL, Colors.FAIL, Colors.FAIL]
list_of_testcase = dynamic_runner.get_tc_list()
if not list_of_testcase:
print_runner_output(Colors.FAIL + 'No testcase Found !!!\nPlease, Check command parameter(s)' + Colors.ENDC)
return
if save_verdict:
list_of_testcase = self.remove_pass_tc(list_of_testcase, verdict_file_path)
fp = None
if save_verdict:
fp = open(verdict_file_path, 'a')
for testcase in list_of_testcase:
if fp:
text = 'verdict | '
text += testcase.binary_name + ' |'
text += testcase.suite_name + ' |'
text += testcase.tc_name + ' |'
text += testcase.package_name + ' |'
text += 'in_queue'
fp.write(text + '\n')
fp.flush()
tc_utility_action = TcUtilityAction(dynamic_runner)
while list_of_testcase:
print_runner_output('{} testcase(s) needed to be run'.format(len(list_of_testcase)))
for testcase in list_of_testcase:
result = 1
print_runner_output('Executing ' + testcase.binary_name + ' ' + testcase.package_name + '.' + testcase.suite_name + '.' + testcase.tc_name)
try:
command, result = self.run_selected_testcase (testcase.binary_name, testcase.suite_name, testcase.tc_name, testcase.package_name, dynamic_runner, tc_utility_action)
print_runner_output('result: {}'.format(result))
if result == 0:
testcase.increase_pass_count()
elif result == 2:
testcase.increase_timeout_count()
testcase.increase_fail_count()
else:
testcase.increase_fail_count()
testcase.increase_total_count()
print_runner_output('increase tc verdict count')
except:
show_exeception()
break
print_runner_output(color_types[result] + '[ ' + 'Result of ' + testcase.suite_name + '.' + testcase.tc_name + ': ' + verdict_types[result] + ' ]' + Colors.ENDC)
if fp:
text = 'verdict | '
text += testcase.binary_name + ' |'
text += testcase.suite_name + ' |'
text += testcase.tc_name + ' |'
text += testcase.package_name + ' |'
text += verdict_types[result]
fp.write(text + '\n')
fp.flush()
if result == 4:
return
list_of_testcase[:] = [x for x in list_of_testcase if not x.is_execution_complete()]
if fp:
fp.close()
print_runner_output("### Test Is Done!!")
def download_fw_output_file(self, output_file):
pull_command = self.get_platform_command_prefix(
) + ' pull ' + self.app_path + '/' + output_file + ' .'
print_runner_output(pull_command)
os.system(pull_command)