def generate_groups(self,
output_filename="egl_lists.txt",
pull_location=helper_scripts_gfx):
"""
This method generates the groups that are to be ran for the EGL 1.4 API level validation
The list is being generated in the '/scripts/'path and it contains all of the tests contained
in suits for the binary.
Of this list, a dictionary will be created containing only the egl lists.
Parameters:
output_filename: the name of the file that will contain all the tests.
Since this more of an auxiliary method, a mock name is already provided. If a user
wishes to make use of this method, the user can override the parameter when calling it.
pull_location: the full path where the list file will be pulled.
Analogue to the previous parameter description, this too can be overriden by the user.
The default is in the '/scripts/' subfolder of the TestCases in ACS.
"""
#first, we generate the lists
generate_cmd = self._adb_bin + "-generateTestList " + self._dut_location + output_filename
generate_result, generate_output = self._aux.send_cmd(generate_cmd, 60)
#now, we pull the full lists on the host
if generate_result == Global.FAILURE:
return generate_result, generate_output
else:
filepath = PathManager.absjoin(pull_location, output_filename)
pull_result, pull_output = self._device.pull(
self._dut_location + output_filename, pull_location, 60)
dict_result, test_dict = self._populate_dict_from_file(filepath)
return dict_result, test_dict
def _run_non_certification_mode(self, group_list_file, timeout):
"""
Method for running a list of tests from a group on all config IDs and all Window Sizes
ConfigID is in the range (1,11) without 8
This method takes 2 parameters:
group_list_file: either a file or a string for the tests to be ran
timeout: integer value, given in seconds
the timeout parameter for methods that interact via adb with the DUT
"""
window_sizes = ['ws_64_64', 'ws_113_47', 'fbo1', 'fbo2']
result, output = Global.BLOCKED, "The run for group lists has not started, blocked at the %s sequence" % group_list_file
if os.path.isfile(group_list_file):
group_list_file2 = PathManager.absjoin(self._helper_scripts_gfx,
group_list_file)
read_groups = open(group_list_file2, 'rt')
for group_file in read_groups:
result, output = self._run_all_configs(window_sizes,
group_file, timeout)
read_groups.close()
else:
group_list_file = os.path.basename(group_list_file)
result, output = self._run_all_configs(window_sizes,
group_list_file, timeout)
return result, output
def __init__(self, device, device_logdir, report_path, activity_name):
"""
Constructor parameters
"""
self._device = device
self._device_logdir = device_logdir
self._report_path = report_path
self._default_logging_dir = '/sdcard/' #this is the log directory that the APK has as default and cannot be overwritten in Native mode
self._activity_name = activity_name
self._logger = LOGGER_TEST_SCRIPT
logcat_cmd_line = self._device.get_config(
"logcatCmdLine", "adb shell logcat -v threadtime")
self._logcat_analyzer = LogCatReaderThread(
self._device,
logger=self._logger,
enable_writer=False,
logcat_cmd_line=logcat_cmd_line)
# self._trigger_message = "Alive"
# self._trigger_message= 'am_finish_activity'
self._trigger_message = 'org.khronos.gl_cts/org.khronos.cts.%sActivity,proc died without state saved' % self._activity_name
self._helper_scripts_path = 'OTC/TC/ACS/Graphics/Graphics/scripts'
self._helper_scripts_gfx = PathManager.absjoin(
Khronos2.TEST_SUITES_PATH, self._helper_scripts_path)
if self._device_logdir is not None:
self._pull_path = self._default_logging_dir + self._device_logdir
def __init__(self, tc_name, attr_override, tc_order, device,
trigger_message, logcat_cmd_line, process_name,
secondary_test_report, helper_scripts_path, device_logdir,
activity_name, report_path, cert_mode):
"""
Constructor parameters
"""
# BaseConf.__init__(self)
# TestStepEngine.__init__(self, tc_conf, global_config)
self._device = device
self._logger = LOGGER_TEST_SCRIPT
self._trigger_message = trigger_message
self._logcat_cmd_line = logcat_cmd_line
self._process_name = process_name
self._tc_report = secondary_test_report
self._logcat_analyzer = LogCatReaderThread(
self._device,
logger=self._logger,
enable_writer=False,
logcat_cmd_line=logcat_cmd_line)
self._default_logging_dir = '/sdcard/' #this is the log directory that the APK has as default and cannot be overwritten in Native mode
self._helper_scripts_gfx = PathManager.absjoin(
Paths.TEST_SUITES, helper_scripts_path
) #this path should be paken out of the Paths...
self._device_logdir = device_logdir
self._activity_name = activity_name
self._report_path = report_path
self._cert_mode = cert_mode
self._tc_name = tc_name
self._tc_order = tc_order
def pull_results_gles1(self, local_path, timeout):
"""
"""
pull_path = PathManager.absjoin(local_path, self._log_folder)
result, output = self._device.pull(self._logs_location, pull_path,
timeout)
return result, output
def _parse_folder_content(self):
results_list = []
parser = BatchResultParser()
for root, dirs, files in os.walk(r'%s' % self._report_path):
for fileu in files:
file2 = PathManager.absjoin(self._report_path, fileu)
if fileu.endswith('.qpa'):
results = parser.parseFile(file2)
for result in results:
results_list.append(result.name + ',' +
result.statusCode)
else:
pass
return results_list
def __init__(self, device, report_path, logs_folder):
"""
Constructor
"""
self._device = device
self._dut_location = "data/app/"
# self._log_folder = 'gles1_logs'
self._log_folder = logs_folder
# self._logs_location = PathManager.absjoin(self._dut_location, self._log_folder)
self._logs_location = self._dut_location + self._log_folder
self._report_path = report_path
self._logs_in_report = PathManager.absjoin(self._report_path,
self._log_folder)
self._aux = Auxiliary()
def extract_schema(cls, node, schema_folder, file_only=True):
"""
Class Method which extracts the XML Schema file reference if so.
:param node: The XML Node or the XML file to be parsed into a node
:type node: etree.Element
:param schema_folder: If provided the value is concatenated to the schema basename if available
:type schema_folder: str
:param file_only: If set to True the schema is not loaded
:type file_only: bool
:return: Either the schema filename or its corresponding XML instance (
:rtype: str or etree.XMLSchema
"""
schema_files = []
if isinstance(node, basestring) and path.isfile(node):
node = etree.parse(node).getroot()
schema_file = node.attrib.get(cls.SCHEMA_LOCATION_KEY)
if schema_file and schema_folder and path.isdir(schema_folder):
schema_file = PathMgr.absjoin(schema_folder,
path.basename(schema_file))
schema_files = os.listdir(schema_folder)
try:
self_name = node.base
except AttributeError:
self_name = "Unknown"
if not schema_file or not path.isfile(schema_file):
_error(("Your Configuration file : {0} does not define an"
" 'xsi:noNamespaceSchemaLocation' in root element!"
).format(self_name))
if path.basename(schema_file) not in schema_files:
_error(
"Your Device Model file : {0} does not define a Valid Schema"
" 'xsi:noNamespaceSchemaLocation' in root element!").format(
self_name)
if file_only:
return schema_file
try:
return etree.XMLSchema(etree.parse(schema_file))
except etree.Error as schema_error:
_error(schema_error.message)
def _run_non_certification_mode(self, group_list_file, timeout):
"""
Method for running a list of tests from a group on all config IDs and all Window Sizes
ConfigID is in the range (1,11) without 8
"""
print "I am suppose to treat al the other situations in which the cert_modes fail miserably cuz of the APK"
window_sizes = ['ws_64_64', 'ws_113_47', 'fbo1', 'fbo2']
group_list_file2 = PathManager.absjoin(self._helper_scripts_gfx,
group_list_file)
read_groups = open(group_list_file2, 'rt')
print "==============="
print "Pathul fisierului din care execut grupuri de teste: %s " % read_groups
result, output = Global.BLOCKED, "The run for group lists has not started, blocked at the %s sequence" % group_list_file
#walk through all the tests in a groups' file
for group_list in read_groups:
print "\n\n Grupul meu: ++%s~~\n" % group_list
#~ # run all groups ws = 64 x 64 config-id 1
result, output = self._single_run(group_list, window_sizes[0], 1,
timeout)
# run all CTS.gtf.* ws = 113 x 47 config-id 1
if "CTS.gtf" in group_list:
result, output = self._single_run(group_list, window_sizes[1],
1, timeout)
# run CTS.gtf.* fbo1
elif "CTS.gtf" in group_list:
result, output = self._single_run(group_list, window_sizes[2],
1, timeout)
# run CTS.gtf.* fbo2
elif "CTS.gtf" in group_list:
result, output = self._single_run(group_list, window_sizes[3],
1, timeout)
# run ws = 64 x 64 all configIDs 2-7, 9-11
elif "CTS.gtf" in group_list or "ES31" in group_list:
for cfgID in [2, 3, 4, 5, 6, 7, 9, 10, 11]:
result, output = self._single_run(group_list,
window_sizes[1], cfgID,
timeout)
# run ws = 113 x 47 all configIDs 2-7, 9-11
elif "CTS.gtf" in group_list:
for cfgID in [2, 3, 4, 5, 6, 7, 9, 10, 11]:
result, output = self._single_run(group_list, "ws_113_47",
cfgID, timeout)
read_groups.close()
return result, output
def _res_to_csv(self, result_filename):
"""
Taken from the sources sent by the Graphics devs
This method will read the QPA results and write them into a CSV file
"""
parser = BatchResultParser()
file_to_parse = PathManager.absjoin(self._report_path, result_filename)
results = parser.parseFile(file_to_parse)
write_in_file = file_to_parse.strip('.qpa') + '.csv'
result_file_csv = open(write_in_file, 'wb')
result_writer = csv.writer(result_file_csv,
delimiter=',',
quoting=csv.QUOTE_NONE)
for result in results:
result_writer.writerow([result.name, result.statusCode])
result_file_csv.close()
return write_in_file
def parse_file_gles1(self, filename):
results_list = []
with open(PathManager.absjoin(self._logs_in_report,
filename)) as result_file:
all_lines = [line.strip().replace(".", "") for line in result_file]
if 'conform' not in filename:
test_prefix = ''
else:
test_prefix = 'conform: '
for line in all_lines:
if 'conform' in filename:
test_name = test_prefix + filename.replace("_log.txt", "")
k = 0
if "SUMMARY:" in line:
k += 1
if 'NO tests failed' in line:
results_list.append(test_name + '$' + 'passed')
elif k == 3 and "test failed" in line:
results_list.append(test_name + '$' + 'failed')
elif "TESTLIST" in filename:
if 'test' in line:
test_name = test_prefix + line.split('test')[0]
results_list.append(test_name + '$' +
line.split('test')[1])
else:
if 'passed' in line or 'failed' in line:
if 'primtest' not in line and 'test' in line:
test_name = test_prefix + line.split('test')[0]
results_list.append(test_name + '$' +
line.split('test')[1])
elif 'primtest' in line:
test_name = test_prefix + line.split(
' (single precision) ')[0]
results_list.append(
test_name + '$' +
line.split(' (single precision) ')[1])
else:
test_name = test_prefix + line.split(' ')[0]
results_list.append(test_name + '$' +
line.split(' ')[1])
return results_list
def _res_to_csv(self, result_filename):
"""
Taken from the sources sent by the Graphics devs
This method will read the QPA results and write them into a CSV file
"""
parser = BatchResultParser()
file_to_parse = PathManager.absjoin(self._report_path, result_filename)
results = parser.parseFile(file_to_parse)
write_in_file = file_to_parse + '.csv'
result_file_csv = open(write_in_file, 'wb')
result_writer = csv.writer(result_file_csv,
delimiter=',',
quoting=csv.QUOTE_NONE)
for result in results:
print '\n\n Asta are trebui sa scriu in csv:'
print result.name, result.statusCode
result_writer.writerow([result.name, result.statusCode])
result_file_csv.close()
# print "\n\nIn _res_to_csv"
# print write_in_file
# print os.path.dirname(write_in_file.name)
return write_in_file
class Khronos2(object):
TEST_SUITES = 'acs_test_suites'
TEST_SUITES_PATH = PathManager.absjoin(PathManager.ACS_TEST_SCRIPTS_DIR,
TEST_SUITES)
def __init__(self, device, device_logdir, report_path, activity_name):
"""
Constructor parameters
"""
self._device = device
self._device_logdir = device_logdir
self._report_path = report_path
self._default_logging_dir = '/sdcard/' #this is the log directory that the APK has as default and cannot be overwritten in Native mode
self._activity_name = activity_name
self._logger = LOGGER_TEST_SCRIPT
logcat_cmd_line = self._device.get_config(
"logcatCmdLine", "adb shell logcat -v threadtime")
self._logcat_analyzer = LogCatReaderThread(
self._device,
logger=self._logger,
enable_writer=False,
logcat_cmd_line=logcat_cmd_line)
# self._trigger_message = "Alive"
# self._trigger_message= 'am_finish_activity'
self._trigger_message = 'org.khronos.gl_cts/org.khronos.cts.%sActivity,proc died without state saved' % self._activity_name
self._helper_scripts_path = 'OTC/TC/ACS/Graphics/Graphics/scripts'
self._helper_scripts_gfx = PathManager.absjoin(
Khronos2.TEST_SUITES_PATH, self._helper_scripts_path)
if self._device_logdir is not None:
self._pull_path = self._default_logging_dir + self._device_logdir
def _execute_command(self, cmd):
"""
This private method offers a way of executing a command during the ACS run by calling the AcsSubprocess method of execute_sync(timeout)
We use this in order to watch the result execution in a separate synchronous subprocess
"""
my_process, stdout = AcsSubprocess(cmd,
LOGGER_TEST_SCRIPT,
max_empty_log_time=1,
silent_mode=False).execute_sync(200)
return my_process, stdout
def _check_process_alive(self, process_name, timeout):
"""
A method that greps for our activity to check if it is still alive. In addition, it also searches for the
SIGSEGV message in the logcat, in order to identify a process that has been terminated before it's normal
finish
"""
start = 0
cmd_header = "adb shell ps | grep "
self._logcat_analyzer.add_trigger_message(self._trigger_message)
self._logcat_analyzer.start()
result, msg = Global.BLOCKED, "Did not start watching the process' life-cycle"
send_cmd = cmd_header + process_name
while start < timeout:
my_proc, stdout = self._execute_command(send_cmd)
if 'khronos' not in stdout:
if self._logcat_analyzer.get_message_triggered_status(
self._trigger_message):
self._logger.debug("Caught the activity being terminated")
result, msg = Global.FAILURE, "The activity did not finish before the process died"
else:
result, msg = Global.SUCCESS, "The activity finished its execution and properly stopped"
return result, msg
else:
result, msg = Global.SUCCESS, "The %s process is alive" % process_name
start += 0.1
return result, msg
def _pull_results(self, remote_path, local_path, special_string, timeout):
"""
Pull the results, whatever the metohds generate and need
Extending(not in a pretty way) _device.pull()
"""
if special_string != '':
full_remote_path = remote_path + "/" + special_string
mock_dir = "mock"
full_mock_path = remote_path + "/" + mock_dir
adb_mkdir_mock = "adb shell mkdir %s/%s" % (self._pull_path,
mock_dir)
status_mkdir, msg_mkdir = self._device.run_cmd(
adb_mkdir_mock, timeout)
adb_cmd_mv = "adb shell mv %s %s" % (full_remote_path,
full_mock_path)
status_mv, msg_mv = self._device.run_cmd(adb_cmd_mv, timeout)
status, err_msg = self._device.pull(full_mock_path, local_path,
timeout)
elif special_string == '':
status, err_msg = self._device.pull(remote_path, local_path,
timeout)
return status, err_msg
def _res_to_csv(self, result_filename):
"""
Taken from the sources sent by the Graphics devs
This method will read the QPA results and write them into a CSV file
"""
parser = BatchResultParser()
file_to_parse = PathManager.absjoin(self._report_path, result_filename)
results = parser.parseFile(file_to_parse)
write_in_file = file_to_parse.strip('.qpa') + '.csv'
result_file_csv = open(write_in_file, 'wb')
result_writer = csv.writer(result_file_csv,
delimiter=',',
quoting=csv.QUOTE_NONE)
for result in results:
result_writer.writerow([result.name, result.statusCode])
result_file_csv.close()
return write_in_file
def _parse_folder_content(self):
results_list = []
parser = BatchResultParser()
for root, dirs, files in os.walk(r'%s' % self._report_path):
for fileu in files:
file2 = PathManager.absjoin(self._report_path, fileu)
if fileu.endswith('.qpa'):
results = parser.parseFile(file2)
for result in results:
results_list.append(result.name + ',' +
result.statusCode)
else:
pass
return results_list
def write_to_csv(self, result_list, csv_file):
"""
"""
result_file_csv = open(csv_file, 'wb')
result_writer = csv.writer(result_file_csv,
delimiter=',',
quoting=csv.QUOTE_NONE)
for result in result_list:
element = result.split(",")
result_writer.writerow([element[0], element[1]])
result_file_csv.close()
return csv_file
def extract_results(self, tc_name, tc_order, tc_report,
result_filename_val):
"""
compute results from gfx csv file test results
- add each test result to a secondary report
:param result_filename: gfx log output and it's location
:type file_path: str
:rtype: str
:return: output message
"""
test_executed = 0
result = Global.FAILURE
output = ""
result_filename = result_filename_val
if result_filename is None:
self._device.get_logger().error(
"There is no specified csv file from GFX fwk !")
output = "GFX result not detected in the log, No result"
result = Global.FAILURE
elif not os.path.isfile(result_filename):
self._device.get_logger().error(
"Specified GFX result file %s does not exist, cannot compute properly results !"
% (result_filename, ))
output = "GFX result is missing, No result"
result = Global.FAILURE
else:
with open(result_filename, "r") as gfx_report:
csv_content = csv.reader(gfx_report,
delimiter=',',
quotechar='|')
result = Global.SUCCESS
for row in csv_content:
if len(row) >= 2:
if StatusCode.PASS in row[
1] or StatusCode.PASSED in row[1]:
test_executed += 1
tc_report.add_result(row[0],
tc_report.verdict.PASS,
"Test PASS", tc_name,
tc_order)
elif StatusCode.SUPPORTED_REPORTED in row[1]:
test_executed += 1
tc_report.add_result(row[0],
tc_report.verdict.PASS,
"Test status is %s" % row[1],
tc_name, tc_order)
elif StatusCode.NOT_SUPPORTED in row[1]:
continue
else:
result = Global.FAILURE
output = "Some tests have the result: FAIL"
test_executed += 1
tc_report.add_result(
row[0], tc_report.verdict.FAIL,
"Test status is %s, see log output for more details"
% row[1], tc_name, tc_order)
else:
result = Global.FAILURE
output = "CSV seems corrupted, there are less than 2 rows!"
if result != Global.FAILURE:
output = "All tests have the result: PASS"
if not test_executed:
result = Global.FAILURE
output = "No tests have been executed"
return result, output
def extract_results_from_list(self, tc_name, tc_order, tc_report,
results_list):
"""
compute results from gfx csv file test results
- add each test result to a secondary report
:param result_filename: gfx log output and it's location
:type file_path: str
:rtype: str
:return: output message
"""
test_executed = 0
if results_list:
result = Global.SUCCESS
for element in results_list:
if len(element) >= 2:
row = element.split(",")
if StatusCode.PASS in row[1] or StatusCode.PASSED in row[1]:
test_executed += 1
tc_report.add_result(row[0], tc_report.verdict.PASS,
"Test PASS", tc_name, tc_order)
elif StatusCode.SUPPORTED_REPORTED in row[1]:
test_executed += 1
tc_report.add_result(row[0], tc_report.verdict.PASS,
"Test status is %s" % row[1],
tc_name, tc_order)
elif StatusCode.NOT_SUPPORTED in row[1]:
continue
else:
result = Global.FAILURE
output = "Some tests have the result: FAIL"
test_executed += 1
tc_report.add_result(
row[0], tc_report.verdict.FAIL,
"Test status is %s, see log output for more details"
% row[1], tc_name, tc_order)
else:
result = Global.FAILURE
output = "The results list seems to be corrupted"
if result != Global.FAILURE:
output = "All tests have the result: PASS"
if not test_executed:
result = Global.FAILURE
output = "No tests have been executed"
else:
result, output = Global.FAILURE, "The list with the results is empty"
return result, output
def extract_no_secondary_report(self, results_list):
"""
"""
test_executed = 0
test_passed = 0
test_failed = 0
test_not_supported = 0
test_sigsegv = 0
if results_list:
result = Global.SUCCESS
for element in results_list:
if len(element) >= 2:
row = element.split(",")
if StatusCode.PASS in row[1] or StatusCode.PASSED in row[1]:
test_executed += 1
test_passed += 1
elif StatusCode.NOT_SUPPORTED in row[1]:
test_executed += 1
test_not_supported += 1
elif "SIGSEGV" in row[1]:
test_executed += 1
test_sigsegv += 1
else:
result = Global.FAILURE
output = "Some tests have the result: FAIL"
test_executed += 1
test_failed += 1
else:
result = Global.FAILURE
output = "The results list seems to be corrupted"
if result != Global.FAILURE:
output = "All tests have the result: PASS"
if not test_executed:
result = Global.FAILURE
output = "No tests have been executed"
else:
result, output = Global.FAILURE, "The list with the results is empty"
output = "The results look like this: \n %s tests were executed;\n %s tests have passed;\n" \
" %s tests are not supported;\n %s tests have a SIGSEGV result;\n %s tests have failed. " \
"\n Please check with the .csv and .qpa file logs for more details." % (test_executed, test_passed,
test_not_supported, test_sigsegv, test_failed)
return result, output
def pull_and_extract(self, tc_name, tc_order, tc_report):
pull_res, pull_out = self._pull_results(self._pull_path,
self._report_path,
special_string='*.qpa',
timeout=500)
result, output = Global.BLOCKED, "Did not yet start extracting"
for root, dirs, files in os.walk(r'%s' % self._report_path):
for file in files:
if file.endswith('.qpa'):
preliminary_file = self._res_to_csv(file)
result, output = self.extract_results(
tc_name,
tc_order,
tc_report,
result_filename_val=preliminary_file)
else:
pass
return result, output
def pull_and_extract_with_lists(self, tc_name, tc_order, tc_report,
csv_report):
pull_res, pull_out = self._pull_results(self._pull_path,
self._report_path,
special_string='*.qpa',
timeout=500)
result, output = Global.BLOCKED, "Did not yet start extracting"
results_list = self._parse_folder_content()
csv_file = self.write_to_csv(results_list, csv_report)
result, output = self.extract_results_from_list(
tc_name, tc_order, tc_report, results_list)
return result, output
def pull_and_extract_no_secondary(self, csv_report):
"""
"""
pull_res, pull_out = self._pull_results(self._pull_path,
self._report_path,
special_string='*.qpa',
timeout=500)
result, output = Global.BLOCKED, "Did not yet start extracting"
results_list = self._parse_folder_content()
csv_file = self.write_to_csv(results_list, csv_report)
result, output = self.extract_no_secondary_report(results_list)
return result, output
def generate_groups(self):
"""
A simple method for generating the test groups
"""
mock_dir = "mock"
full_mock_path = self._default_logging_dir + "/" + mock_dir
adb_mkdir_mock = "adb shell mkdir %s" % full_mock_path
status_mkdir, msg_mkdir = self._device.run_cmd(adb_mkdir_mock, 20)
groups_cmd = "adb shell am start -n org.khronos.gl_cts/android.app.NativeActivity \
-e cmdLine 'cts --deqp-runmode=txt-caselist'"
pull_sequence = "ES*"
result, output = self._device.run_cmd(groups_cmd, 1500)
group_files = [
"ES2-CTS-groups.txt", "ES3-CTS-groups.txt", "ES31-CTS-groups.txt"
]
if result == Global.SUCCESS:
pull_result, pull_msg = self._pull_results(
self._default_logging_dir, self._helper_scripts_gfx,
pull_sequence, 1500)
if pull_result == Global.FAILURE:
result, output = Global.FAILURE, "Could not pull the groups files from the DUT, please check logs"
else:
for root, dirs, files in os.walk(r'%s/' %
self._helper_scripts_gfx):
for file in files:
for gf in group_files:
tf_d = open(self._helper_scripts_gfx + "/" + file,
'r')
gf_d = open(self._helper_scripts_gfx + "/" + gf,
'w')
group_run = None
for line in tf_d:
if "GROUP" in line:
group_run = line.split(" ")[1]
if ("TEST" in line) and group_run:
gf_d.write(
group_run.strip('\n') + '.*' + '\n')
group_run = None
tf_d.close()
gf_d.close()
result, output = Global.SUCCESS, "Files were generated in the %s folder" \
% self._helper_scripts_gfx
return result, output
def _run_certification_mode_full(self):
"""
Method for running the ES31Activity
cam hardcodata partea cu activity name, merge momentan
"""
adb_run_all = "adb shell am start -n org.khronos.gl_cts/org.khronos.cts.%sActivity -e logdir '%s/%s'" % (
self._activity_name, self._default_logging_dir,
self._device_logdir)
result, output = self._device.run_cmd(adb_run_all, 1500)
return result, output
def single_run(self, group_name, window_size, configID, timeout=300):
"""
This method runs a single test or a group of tests using the NativeActivity from the APK
Parameters:
group_name: the name of the test or group to be ran.
Can either be a full name test or one that includes more tests.
Usage example:
- 1 test: ES3-CTS.info.vendor
- 1 group: ES3-CTS.info.*
window_size: this parameter is the possible window size. The possible values map with the
win_cmd dictionary.
Possible values:
- ws_64_64
- ws_113_47
- fbo1
- fbo2
configID: parameter for the config-id of the test. Possible values are in the range 1-11
timeout: integer value, given in seconds
the timeout parameter for methods that interact via adb with the DUT
"""
# dictionary to keep formatted options for different windows sizes/ fbo
win_cmd = {
"ws_64_64":
"\"cts \
--deqp-surface-width=64 \
--deqp-surface-height=64 \
--deqp-base-seed=1 \
--deqp-surface-type=window ",
"ws_113_47":
"\"cts \
--deqp-surface-width=113 \
--deqp-surface-height=47 \
--deqp-base-seed=2 \
--deqp-surface-type=window ",
"fbo1":
"\"cts \
--deqp-surface-width=64 \
--deqp-surface-height=64 \
--deqp-base-seed=3 \
--deqp-surface-type=window \
--deqp-use-fbo=GL_RGBA8,GL_DEPTH24_STENCIL8,64,max ",
"fbo2":
"\"cts \
--deqp-surface-width=64 \
--deqp-surface-height=64 \
--deqp-base-seed=3 \
--deqp-surface-type=window \
--deqp-use-fbo=GL_RGBA8,GL_DEPTH24_STENCIL8,max,64 "
}
adb_am_start = "adb shell am start -n org.khronos.gl_cts/android.app.NativeActivity -e cmdLine "
# log name for the test result
if "*" in group_name:
log_name = "%s%s/%s_%s_cfgID_%s.qpa" % (
self._default_logging_dir, self._device_logdir,
group_name[0:-2], window_size, configID)
else:
log_name = "%s%s/%s_%s_cfgID_%s.qpa" % (
self._default_logging_dir, self._device_logdir, group_name,
window_size, configID)
# group to test and log file part of the command
cmd_line_arg = "--deqp-case=%s --deqp-log-filename=%s " % (group_name,
log_name)
cmd_ID_arg = "--deqp-gl-config-id=%s\"" % str(configID)
adb_single_run = adb_am_start + win_cmd[
window_size] + cmd_line_arg + cmd_ID_arg
run_result, run_output = self._device.run_cmd(adb_single_run, timeout)
return run_result, run_output
def _run_all_configs(self, window_sizes, group_list, timeout):
"""
This method runs all the possible config-ids for the tests that provide the following
parameters:
window_sizes: a vector with the possible values for window size
group_list: parameter for a test or a group that has to be ran.
This method is called later into the _run_non_certification_mode() method.
This method uses the previous method single_run
timeout: integer value, given in seconds
the timeout parameter for methods that interact via adb with the DUT
"""
#~ # run all groups ws = 64 x 64 config-id 1
result, output = self.single_run(group_list, window_sizes[0], 1,
timeout)
# run all CTS.gtf.* ws = 113 x 47 config-id 1
if "CTS.gtf" in group_list:
result, output = self.single_run(group_list, window_sizes[1], 1,
timeout)
# run CTS.gtf.* fbo1
if "CTS.gtf" in group_list:
result, output = self.single_run(group_list, window_sizes[2], 1,
timeout)
# run CTS.gtf.* fbo2
if "CTS.gtf" in group_list:
result, output = self.single_run(group_list, window_sizes[3], 1,
timeout)
# run ws = 64 x 64 all configIDs 2-7, 9-11
if "CTS.gtf" in group_list or "ES31" in group_list:
for cfgID in [2, 3, 4, 5, 6, 7, 9, 10, 11]:
result, output = self.single_run(group_list, window_sizes[1],
cfgID, timeout)
# run ws = 113 x 47 all configIDs 2-7, 9-11
if "CTS.gtf" in group_list:
for cfgID in [2, 3, 4, 5, 6, 7, 9, 10, 11]:
result, output = self.single_run(group_list, "ws_113_47",
cfgID, timeout)
return result, output
def _run_non_certification_mode(self, group_list_file, timeout):
"""
Method for running a list of tests from a group on all config IDs and all Window Sizes
ConfigID is in the range (1,11) without 8
This method takes 2 parameters:
group_list_file: either a file or a string for the tests to be ran
timeout: integer value, given in seconds
the timeout parameter for methods that interact via adb with the DUT
"""
window_sizes = ['ws_64_64', 'ws_113_47', 'fbo1', 'fbo2']
result, output = Global.BLOCKED, "The run for group lists has not started, blocked at the %s sequence" % group_list_file
if os.path.isfile(group_list_file):
group_list_file2 = PathManager.absjoin(self._helper_scripts_gfx,
group_list_file)
read_groups = open(group_list_file2, 'rt')
for group_file in read_groups:
result, output = self._run_all_configs(window_sizes,
group_file, timeout)
read_groups.close()
else:
group_list_file = os.path.basename(group_list_file)
result, output = self._run_all_configs(window_sizes,
group_list_file, timeout)
return result, output
def _basic_run_method(self, cert_mode, group_list, timeout):
"""
This method asserts which mode will be ran. The reason for it's existence is that, even though now there is only
one method (full_run) that will be called in the UseCase, a second one is in plan that also takes into account
the possible crashes encountered by the APK.
Parameters:
cert_mode: boolean
Whether the certification or partial(non-certification) mode will be used.
group_list: string/file for the non-certificated method.
timeout: integer value, given in seconds
the timeout parameter for methods that interact via adb with the DUT
"""
result, output = Global.BLOCKED, "Execution not start based on cert mode"
if cert_mode == 'True':
result, output = self._run_certification_mode_full()
elif cert_mode == 'False':
result, output = self._run_non_certification_mode(
group_list, timeout)
return result, output
def full_run(self, process_name, cert_mode, group_list, run_type, timeout):
"""
The method at hand calls for the _basic_run_method described above. After the parameters are evaluated and the
activity has started, it launches an ACS subprocess that checks whether the process is still alive. If the
process is no longer alive, the method for pulling the reports is called.
The final result and output are those of the pull_and_extract method.
Parameters:
process_name: string
the name of the process that is being watched.
cert_mode: boolean
group_list:
tc_name:
tc_order:
timeout: integer value, given in seconds
the timeout parameter for methods that interact via adb with the DUT
"""
mock_dir = "mock"
full_mock_path = self._default_logging_dir + mock_dir
adb_mkdir_mock = "adb shell mkdir %s" % full_mock_path
status_mkdir, msg_mkdir = self._device.run_cmd(adb_mkdir_mock, 20)
result, output = self._basic_run_method(cert_mode, group_list, timeout)
alive_bool, alive_msg = self._check_process_alive(
process_name, timeout)
if run_type is not None:
if run_type.lower() is 'dry':
return alive_bool, alive_msg
else:
self._device.run_cmd(
"adb shell rm -rf %s" % self._default_logging_dir, 20)
self._device.run_cmd("adb shell rm -rf %s" % full_mock_path,
20)
status_mkdir, msg_mkdir = self._device.run_cmd(
adb_mkdir_mock, 20)
result, output = self._run_non_certification_mode(
group_list, timeout)
return result, output
else:
return alive_bool, alive_msg
class OGLConform(object):
gfx_scripts_path = 'OTC/TC/ACS/Graphics/Graphics/scripts'
helper_scripts_gfx = PathManager.absjoin(KhronosObj.TEST_SUITES_PATH,
gfx_scripts_path)
def __init__(self, device):
self._device = device
self._dut_location = "data/app/"
self._adb_bin = self._dut_location + 'oglconform '
self._minFmt = '-v 4 -minFmt '
self._aux = Auxiliary()
def feature_supported(self, *param_element_list):
"""
This function checks if the given feature is supported
"""
supported_bool = True
for element in param_element_list:
if 'not' in element.lower():
supported_bool = False
return supported_bool
def treat_diag(self, param_diag, feature_name):
"""
This function treats the conditions containing the -diag option for validating a feature
param_diag: shell -diag option to run and switch. -diag -es1 or -es2. Given by TCParameters
feature_name: feature name to be tested. Given by TCParameters
"""
diag_cmd = self._adb_bin + self._minFmt + param_diag + " | grep '" + feature_name + ":' -A 2"
out_result, out_file = self._aux.send_cmd(diag_cmd, 60)
diag_list = out_file.strip().split('\n')
result = "SUPPORTED AND REPORTED"
if not self.feature_supported(diag_list[1].strip(), diag_list[2]):
self._aux.exit(
'feature is not shown as supported by oglconform -diag')
result = "NOT SUPPORTED AND REPORTED"
return result
def treat_testcase(self, param_test):
"""
This function treats the condition containing the -test option
param_test: test parameter to be given for the -test option
"""
result = ''
list_outcomes = ['Total Passed', 'Total Failed', 'Total Not run']
test_cmd = self._adb_bin + self._minFmt + param_test + ' | tail -n5'
out_result, out_file = self._aux.send_cmd(test_cmd, 60)
if not any(outcome in out_file for outcome in list_outcomes):
loc_splitter = param_test.split('-test ')
tc_name = max(loc_splitter, key=len)
self._aux.exit(tc_name + ' failed because of subcase timeout ' +
"\nERROR is: " + str(out_file))
err_str = 'Error encountered'
if err_str in out_file:
if 'does not contain testcase/group' in err_str:
self._aux.exit(
'Error encountered: %s . Skipping execution to next testcase.'
% err_str)
result = "BLOCKED DEPENDENCY"
test_list = out_file.strip().split('\n')
for l_line in test_list:
if 'Total Failed :' in l_line:
result = "PASSED"
if not ('0' in l_line):
self._aux.exit('%s failed' % str(param_test))
result = "FAILED"
return result
def _populate_dict_from_file(self, tests_filepath):
"""
This is a method that we use in the generate_groups method in order to move our tests in a dictionary
"""
if not os.path.isfile(tests_filepath):
# if not os.stat(tests_filepath).st_size > 0:
return Global.FAILURE, "The path does not point to a file"
else:
with open(tests_filepath, 'r') as test_instance:
test_dict = [
line.strip() for line in test_instance if "egl-" in line
]
test_instance.close()
return Global.SUCCESS, test_dict
def generate_groups(self,
output_filename="egl_lists.txt",
pull_location=helper_scripts_gfx):
"""
This method generates the groups that are to be ran for the EGL 1.4 API level validation
The list is being generated in the '/scripts/'path and it contains all of the tests contained
in suits for the binary.
Of this list, a dictionary will be created containing only the egl lists.
Parameters:
output_filename: the name of the file that will contain all the tests.
Since this more of an auxiliary method, a mock name is already provided. If a user
wishes to make use of this method, the user can override the parameter when calling it.
pull_location: the full path where the list file will be pulled.
Analogue to the previous parameter description, this too can be overriden by the user.
The default is in the '/scripts/' subfolder of the TestCases in ACS.
"""
#first, we generate the lists
generate_cmd = self._adb_bin + "-generateTestList " + self._dut_location + output_filename
generate_result, generate_output = self._aux.send_cmd(generate_cmd, 60)
#now, we pull the full lists on the host
if generate_result == Global.FAILURE:
return generate_result, generate_output
else:
filepath = PathManager.absjoin(pull_location, output_filename)
pull_result, pull_output = self._device.pull(
self._dut_location + output_filename, pull_location, 60)
dict_result, test_dict = self._populate_dict_from_file(filepath)
return dict_result, test_dict
def execute_conformance(self, result_filename):
"""
This method executes the EGL 1.4 conformance, based on all the egl suites generated on the spot
with the generate_groups method.
"""
groups_result, groups_output = self.generate_groups()
for test in groups_output:
if "es1" in test:
test_string = "-es1 -test " + test
return self.treat_testcase("-es1 -test " + test), test
else:
test_string = "-es2 -test " + test
test_result = self.treat_testcase(test_string)
write_report(test, test_result, result_filename)
def __init__(self, tc_name, global_config):
"""
Constructor
"""
UseCaseBase.__init__(self, tc_name, global_config)
self._oglconform = OGLConform(self._device)
self._report_pathname = self._device.get_report_tree().get_report_path(
)
self._output_filename = "%s/%s.csv" % (
self._report_pathname, os.path.basename(self.get_name()))
self._output_filename_khronos = "%s/khronos_%s.csv" % (
self._report_pathname, os.path.basename(self.get_name()))
# get TestCase parameters
self._feature_name = str(
self._tc_parameters.get_param_value("FEATURE_NAME", ""))
self._diag_parameter = str(
self._tc_parameters.get_param_value("DIAG_PARAMETER",
"")).split(";")
self._test_parameter = str(
self._tc_parameters.get_param_value("TEST_PARAMETER",
"")).replace("\n",
"").split(";")
self._khronos_parameter = str(
self._tc_parameters.get_param_value("KHRONOS_PARAMETER",
"")).split(";")
self._remove_parameter = str(
self._tc_parameters.get_param_value("REMOVE_PARAMS",
"")).replace("\n",
"").split(";")
self._device_logdir = self._tc_parameters.get_param_value(
"DEVICE_LOGDIR")
self._cert_mode = self._tc_parameters.get_param_value("CERT_MODE")
self._run_type = 'DRY'
self.__tc_report = SecondaryTestReport(
self._device.get_report_tree().get_report_path())
self._adb_remove = "adb shell rm -rf /data/app/"
self._proc_name = "org.khronos.gl_cts"
self._report_path = self._device.get_report_tree().get_report_path()
self._khronos_object = KhronosApk(self._device, self._device_logdir,
self._report_path, None)
self.original_results_gfx_location = os.path.join(
os.path.expanduser("~"), "test_output")
self.acs_gfx_results = os.path.join(
self._device.get_report_tree().get_report_path(),
"gfx_results.zip")
self._helper_scripts_path = 'OTC/TC/ACS/Graphics/Graphics/scripts'
self._scripts_path = PathManager.absjoin(KhronosApk.TEST_SUITES_PATH,
self._helper_scripts_path)
self._activity_name = self._tc_parameters.get_param_value(
"ACTIVITY_NAME")
self._default_logging_dir = "/sdcard/"
name_list = self.get_name().split("/")
tcs_name = name_list[len(name_list) - 1]
self._logs_folder_name = "gles_%s_logs" % tcs_name
self._pull_logs_path = PathManager.absjoin(self._report_path,
self._logs_folder_name)
self._output_filename = "%s/%s.csv" % (
self._report_pathname, os.path.basename(self.get_name()))
self._output_filename_khronos = "%s/khronos_%s.csv" % (
self._report_pathname, os.path.basename(self.get_name()))
self._khronos = KhronosBin(self._device, self._report_pathname,
self._logs_folder_name)
# self._khronos_object = KhronosApk(self._device, self._device_logdir, self._report_path, self._activity_name)
self._khronos_object = KhronosApk(self._device, self._device_logdir,
self._pull_logs_path,
self._activity_name)
def __init__(self, tc_name, global_config):
"""
Constructor
"""
UseCaseBase.__init__(self, tc_name, global_config)
self._logger = LOGGER_TEST_SCRIPT
logcat_cmd_line = self._device.get_config(
"logcatCmdLine", "adb shell logcat -v threadtime")
self._logcat_extract = os.path.join(Folders.REPORTS,
"parserlogcat.log")
self._logcat_analyzer = LogCatReaderThread(
self._device,
logger=self._logger,
enable_writer=False,
logcat_cmd_line=logcat_cmd_line)
self._proc_name = "org.khronos.gl_cts"
#parameter that says how you want to run the tests: dry or full
#dry: run once, if it fails, log the result, close execution
#full: if it fails, re-run from the last entry, log the results as you go
self._run_type = self._tc_parameters.get_param_value("RUN_TYPE")
#this parameter says if the run is in certification mode or not
self._cert_mode = self._tc_parameters.get_param_value("CERT_MODE")
#give the location on the device for the logs
self._device_logdir = self._tc_parameters.get_param_value(
"DEVICE_LOGDIR")
#initializing the secondary test results
#this is also where the intermediate and so on results must be pulled
self._report_path = self._device.get_report_tree().get_report_path()
self.__tc_report = SecondaryTestReport(self._report_path)
#the sdcard is the default logging directory for the ES APK
self._default_logging_dir = "/sdcard/"
self._helper_scripts = 'OTC/TC/ACS/Graphics/Graphics/scripts'
#this is also where the group and test lists must be pulled
self._helper_scripts_gfx = PathManager.absjoin(
KhronosApk.TEST_SUITES_PATH, self._helper_scripts)
self._activity_name = self._tc_parameters.get_param_value(
"ACTIVITY_NAME")
self._group_list_filename = self._tc_parameters.get_param_value(
"GROUPS_FILENAME")
if self._tc_parameters.get_param_value("SINGLE_TEST_PARAM"):
self._single_test_parameter = str(
self._tc_parameters.get_param_value(
"SINGLE_TEST_PARAM")).split(";")
else:
self._single_test_parameter = None
name_list = self.get_name().split("/")
tcs_name = name_list[len(name_list) - 1]
self._logs_folder_name = "gles_%s_logs" % tcs_name
self._pull_logs_path = PathManager.absjoin(self._report_path,
self._logs_folder_name)
if self._group_list_filename:
self._group_list = PathManager.absjoin(self._helper_scripts_gfx,
self._group_list_filename)
else:
self._group_list = self._helper_scripts_gfx
# self._khronos_object = KhronosApk(self._device, self._device_logdir, self._report_path, self._activity_name)
self._khronos_object = KhronosApk(self._device, self._device_logdir,
self._pull_logs_path,
self._activity_name)
self._output_filename_khronos = "%s/khronos_%s.csv" % (
self._report_path, os.path.basename(self.get_name()))
def extract_results(self, result_filename_val):
"""
#TODO: this method should really be taken out of a usecase, in an utilities file...
compute results from gfx csv file test results
- add each test result to a secondary report
:param result_filename: gfx log output and it's location
:type file_path: str
:rtype: str
:return: output message
"""
print "==============="
print "Ce prinde get_name() si dupa tc_order:"
print str(self._tc_name)
print self._tc_order
# print self._tc_name.get_name()
print "==============="
test_executed = 0
result = Global.FAILURE
output = ""
result_filename = PathManager.absjoin(self._report_path,
result_filename_val)
if result_filename is None:
self._device.get_logger().error(
"There is no specified csv file from GFX fwk !")
output = "GFX result not detected in the log, No result"
result = Global.FAILURE
elif not os.path.isfile(result_filename):
self._device.get_logger().error(
"Specified GFX result file %s does not exist, cannot compute properly results !"
% (result_filename, ))
output = "GFX result is missing, No result"
result = Global.FAILURE
else:
with open(result_filename, "r") as gfx_report:
csv_content = csv.reader(gfx_report,
delimiter=',',
quotechar='|')
result = Global.SUCCESS
for row in csv_content:
if len(row) >= 2:
if StatusCode.PASS in row[1]:
test_executed += 1
self._tc_report.add_result(
row[0], self._tc_report.verdict.PASS,
"Test PASS", self._tc_name, self._tc_order)
elif StatusCode.NOT_SUPPORTED in row[1]:
# TEST SHOULD NOT BE RUN FOR THIS PLATFORM, so pass to another one
continue
else:
result = Global.FAILURE
output = "Some tests are FAIL"
test_executed += 1
self._tc_report.add_result(
row[0], self._tc_report.verdict.FAIL,
"Test status is %s, see log output for more details"
% row[1], self._tc_name, self._tc_order)
else:
result = Global.FAILURE
output = "CSV seems corrupted, there are less than 2 rows!"
if result != Global.FAILURE:
output = "All tests are PASS"
if not test_executed:
result = Global.FAILURE
output = "No tests have been executed"
return result, output