def testOnlyResultLines(self):
"""Test when the stdout is only Result lines."""
result = telemetry_runner.TelemetryResult(
exit_code=0, stdout=self.SAMPLE_RESULT_LINES)
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.SUCCESS_STATUS)
self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data)
def testBadCharactersInResultStringComponents(self):
"""Test bad characters are cleaned up in RESULT string components."""
stdout = (
'RESULT average_commit_time_by_url!: '
'http___www.^^ebay.com= 8.86528 ms\n'
'RESULT CodeLoad*: CodeLoad= 6343 score\n'
'RESULT ai-astar: ai-astar= '
'[614,527,523,471,530,523,577,625,614,538] ~~ms\n'
'RESULT !!graph_name: &&test_name= {3.14, 0.98} units!')
expected_perf_data = [
{'graph': 'average_commit_time_by_url_',
'trace': 'http___www.__ebay.com',
'units': 'ms', 'value': 8.86528},
{'graph': 'CodeLoad_', 'trace': 'CodeLoad',
'units': 'score', 'value': 6343},
{'graph': 'ai-astar', 'trace': 'ai-astar',
'units': '__ms', 'value': 554.2},
{'graph': '__graph_name', 'trace': '__test_name',
'units': 'units_', 'value': 3.14}]
result = telemetry_runner.TelemetryResult(exit_code=0, stdout=stdout,
stderr='')
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.SUCCESS_STATUS)
self.assertEquals(expected_perf_data, result.perf_data)
def testInfoBeforeResultLines(self):
"""Test when there is info before the Result lines."""
stdout = ('Pages: [http://www.google.com, http://www.facebook.com]\n' +
self.SAMPLE_RESULT_LINES)
stderr = 'WARNING: Page failed to load http://www.facebook.com\n'
result = telemetry_runner.TelemetryResult(exit_code=1, stdout=stdout,
stderr=stderr)
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.WARNING_STATUS)
self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data)
def testOnlyResultLinesWithWarnings(self):
"""Test when the stderr has Warnings."""
stdout = self.SAMPLE_RESULT_LINES
stderr = ('WARNING: Page failed to load http://www.facebook.com\n'
'WARNING: Page failed to load http://www.yahoo.com\n')
result = telemetry_runner.TelemetryResult(exit_code=2, stdout=stdout,
stderr=stderr)
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.WARNING_STATUS)
self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data)
def testInfoAfterResultLines(self):
"""Test when there is info after the Result lines."""
stdout = (self.SAMPLE_RESULT_LINES + '\n'
'stderr:WARNING:root:Found (system), but you do not have '
'a DISPLAY environment set.\n\n'
'04/16 12:51:23.312 DEBUG|telemetry_:0139|')
result = telemetry_runner.TelemetryResult(exit_code=0, stdout=stdout,
stderr='')
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.SUCCESS_STATUS)
self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data)
def testNoResultLines(self):
"""Test when Result lines are missing from stdout."""
stdout = ('Pages: [http://www.google.com, http://www.facebook.com]\n'
'stderr:WARNING:root:Found (system), but you do not have '
'a DISPLAY environment set.\n\n'
'04/16 12:51:23.312 DEBUG|telemetry_:0139|')
result = telemetry_runner.TelemetryResult(exit_code=0, stdout=stdout,
stderr='')
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.SUCCESS_STATUS)
self.assertEquals([], result.perf_data)
def testOnlyResultLinesWithWarningsAndTraceback(self):
"""Test when the stderr has Warnings and Traceback."""
stdout = self.SAMPLE_RESULT_LINES
stderr = ('WARNING: Page failed to load http://www.facebook.com\n'
'WARNING: Page failed to load http://www.yahoo.com\n'
'Traceback (most recent call last):\n'
'File "../../utils/unittest_suite.py", line 238, in '
'<module>\n'
'main()')
result = telemetry_runner.TelemetryResult(exit_code=2, stdout=stdout,
stderr=stderr)
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.FAILED_STATUS)
self.assertEquals(self.EXPECTED_PERF_DATA, result.perf_data)
def testCleanupUnitsString(self):
"""Test that special characters in units strings are cleaned up."""
result = telemetry_runner.TelemetryResult()
self.assertEquals(result._cleanup_units_string('score/unit'),
'score_per_unit')
self.assertEquals(result._cleanup_units_string('score / unit'),
'score__per__unit')
self.assertEquals(result._cleanup_units_string('%'),
'percent')
self.assertEquals(result._cleanup_units_string('unit%'),
'unitpercent')
self.assertEquals(result._cleanup_units_string('^^un!ts##'),
'__un_ts__')
self.assertEquals(result._cleanup_units_string('^^un!ts##/time %'),
'__un_ts___per_time_percent')
def run_once(self, args, client_ip='', dut=None):
"""
Run a single telemetry test.
@param args: A dictionary of the arguments that were passed
to this test.
@param client_ip: The ip address of the DUT
@param dut: The autotest host object representing DUT.
@returns A TelemetryResult instance with the results of this execution.
"""
test_name = args['test']
test_args = ''
if 'test_args' in args:
test_args = args['test_args']
# Decide whether the test will run locally or by a remote server.
if 'run_local' in args and args['run_local'].lower() == 'true':
# The telemetry scripts will run on DUT.
_ensure_deps(dut, test_name)
format_string = ('python %s --browser=system %s %s')
command = format_string % (os.path.join(CLIENT_CHROME_ROOT,
RUN_BENCHMARK),
test_args, test_name)
runner = dut
else:
# The telemetry scripts will run on server.
format_string = ('python %s --browser=cros-chrome --remote=%s '
'%s %s')
command = format_string % (os.path.join(_find_chrome_root_dir(),
RUN_BENCHMARK),
client_ip, test_args, test_name)
runner = utils
# Run the test.
stdout = StringIO.StringIO()
stderr = StringIO.StringIO()
try:
logging.info('CMD: %s', command)
result = runner.run(command, stdout_tee=stdout, stderr_tee=stderr,
timeout=TELEMETRY_TIMEOUT_MINS*60)
exit_code = result.exit_status
except error.CmdError as e:
logging.debug('Error occurred executing telemetry.')
exit_code = e.result_obj.exit_status
raise error.TestFail('An error occurred while executing '
'telemetry test.')
finally:
stdout_str = stdout.getvalue()
stderr_str = stderr.getvalue()
stdout.close()
stderr.close()
# Parse the result.
logging.debug('Telemetry completed with exit code: %d.'
'\nstdout:%s\nstderr:%s', exit_code,
stdout_str, stderr_str)
logging.info('Telemetry completed with exit code: %d.'
'\nstdout:%s\nstderr:%s', exit_code,
stdout_str, stderr_str)
result = telemetry_runner.TelemetryResult(exit_code=exit_code,
stdout=stdout_str,
stderr=stderr_str)
result.parse_benchmark_results()
for data in result.perf_data:
self.output_perf_value(description=data['trace'],
value=data['value'],
units=data['units'],
graph=data['graph'])
return result
def testEmptyStdout(self):
"""Test when the test exits with 0 but there is no output."""
result = telemetry_runner.TelemetryResult()
result.parse_benchmark_results()
self.assertEquals(result.status, telemetry_runner.FAILED_STATUS)