def get_filter_option(self, profile, size):
"""Get option of filtering test.
@param profile: The profile to encode into.
@param size: The size of test stream in pair format (width, height).
"""
# Profiles used in blacklist to filter test for specific profiles.
H264 = 1
VP8 = 11
VP9 = 12
blacklist = {
# (board, profile, size): [tests to skip...]
# "board" supports Unix shell-type wildcards.
# Use None for "profile" or "size" to indicate no filter on it.
# It is possible to match multiple keys for board/profile/size
# in the blacklist, e.g. veyron_minnie could match both
# "veyron_*" and "veyron_minnie".
# rk3399 doesn't support HW encode for plane sizes not multiple
# of cache line.
('kevin', None, None): ['CacheLineUnalignedInputTest/*'],
('bob', None, None): ['CacheLineUnalignedInputTest/*'],
# Still high failure rate of VP8 EncoderPerf for veyrons,
# disable it for now. crbug/720386
('veyron_*', VP8, None): ['EncoderPerf/*'],
# Disable mid_stream_bitrate_switch test cases for elm/hana.
# crbug/725087
('elm', None, None): ['MidStreamParamSwitchBitrate/*',
'MultipleEncoders/*'],
('hana', None, None): ['MidStreamParamSwitchBitrate/*',
'MultipleEncoders/*'],
# Around 40% failure on elm and hana 320x180 test stream.
# crbug/728906
('elm', H264, (320, 180)): ['ForceBitrate/*'],
('elm', VP8, (320, 180)): ['ForceBitrate/*'],
('hana', H264, (320, 180)): ['ForceBitrate/*'],
('hana', VP8, (320, 180)): ['ForceBitrate/*'],
}
board = utils.get_current_board()
filter_list = []
for (board_key, profile_key, size_key), value in blacklist.items():
if (fnmatch.fnmatch(board, board_key) and
(profile_key is None or profile == profile_key) and
(size_key is None or size == size_key)):
filter_list += value
if filter_list:
return '-' + ':'.join(filter_list)
return ''
def run_once(self):
"""Entry point of this test."""
# The test is flaky on x86-* boards. Mash doesn't target hardware this
# old, so skip the test. http://crbug.com/679213
# The test is also flaky on nyan_* boards. Temporarily skip the test
# until this can be fixed. http://crbug.com/717275
boards_to_skip = [
'x86-mario', 'x86-alex', 'x86-alex_he', 'x86-zgb', 'x86-zgb_he',
'nyan_big', 'nyan_kitty', 'nyan_blaze'
]
if utils.get_current_board() in boards_to_skip:
logging.warning('Skipping test run on this board.')
return
# GPU info collection via devtools SystemInfo.getInfo does not work
# under mus due to differences in how the GPU process is configured.
# http://crbug.com/669965
mus_browser_args = ['--mus', '--gpu-no-complete-info-collection']
logging.info('Testing Chrome --mus startup.')
with chrome.Chrome(auto_login=False,
extra_browser_args=mus_browser_args):
logging.info('Chrome --mus started and loaded OOBE.')
logging.info('Testing Chrome --mus login.')
with chrome.Chrome(extra_browser_args=mus_browser_args):
logging.info('Chrome login with --mus succeeded.')
def run_once(self):
"""Runs the test."""
self.board = utils.get_current_board()
with chrome.Chrome(extra_browser_args=EXTRA_BROWSER_ARGS,
init_network_controller=True) as cr:
self.start_getusermedia(cr)
self.print_perf_results()
def is_skipping_test(self, codec, is_switchres):
"""Determine whether this test should skip.
@param codec: the codec to be tested, ex. 'vp8', 'vp9', 'h264'.
@param is_switchres: bool, True if using switch resolution video.
"""
blacklist = [
# (board, codec, is_switchres); None if don't care.
# "board" supports Unix shell-type wildcards
# Disable vp8 switchres for nyan devices temporarily due to:
# crbug/699260
('nyan', 'vp8', True),
('nyan_*', 'vp8', True)
]
board = utils.get_current_board()
for entry in blacklist:
if ((entry[0] is None or fnmatch.fnmatch(board, entry[0]))
and (entry[1] is None or codec == entry[1])
and (entry[2] is None or is_switchres == entry[2])):
return True
return False
def run_once(self):
"""Entry point of this test."""
# Mash requires a connected display to start chrome. Chromebox and
# Chromebit devices in the lab run without a connected display.
# Limit this test to devices with a built-in display until we can fix
# mash. http://crbug.com/673561
if site_utils.get_board_type() not in ['CHROMEBOOK', 'CHROMEBASE']:
logging.warning('chrome --mash requires a display, skipping test.')
return
# The test is sometimes flaky on these boards. Mash doesn't target
# hardware this old, so skip the test. http://crbug.com/679213
boards_to_skip = [
'x86-mario', 'x86-alex', 'x86-alex_he', 'x86-zgb', 'x86-zgb_he'
]
if utils.get_current_board() in boards_to_skip:
logging.warning('Skipping test run on this board.')
return
# GPU info collection via devtools SystemInfo.getInfo does not work
# under mash due to differences in how the GPU process is configured.
# http://crbug.com/669965
mash_browser_args = ['--mash', '--gpu-no-complete-info-collection']
logging.info('Testing Chrome --mash startup.')
with chrome.Chrome(auto_login=False,
extra_browser_args=mash_browser_args):
logging.info('Chrome --mash started and loaded OOBE.')
logging.info('Testing Chrome --mash login.')
with chrome.Chrome(extra_browser_args=mash_browser_args):
logging.info('Chrome login with --mash succeeded.')
def run_once(self):
"""Entry point of this test."""
# Flaky on nyan_* boards. http://crbug.com/717275
boards_to_skip = ['nyan_big', 'nyan_kitty', 'nyan_blaze']
if utils.get_current_board() in boards_to_skip:
logging.warning('Skipping test run on this board.')
return
# GPU info collection via devtools SystemInfo.getInfo does not work
# under mash due to differences in how the GPU process is configured
# with mus hosting viz. http://crbug.com/669965
mash_browser_args = [
'--enable-features=Mash', '--disable-features=SingleProcessMash',
'--gpu-no-complete-info-collection'
]
logging.info('Testing Chrome with Mash startup.')
with chrome.Chrome(auto_login=False,
extra_browser_args=mash_browser_args):
logging.info('Chrome with Mash started and loaded OOBE.')
self.__screen_visual_sanity_test()
logging.info('Testing Chrome with Mash login.')
with chrome.Chrome(extra_browser_args=mash_browser_args):
logging.info('Chrome login with Mash succeeded.')
def run_once(self):
"""
Entry point of this test.
"""
self.job.install_pkg(self.dep, 'dep', self.dep_dir)
with service_stopper.ServiceStopper([self.adapter_service]):
cmd = [os.path.join(self.dep_dir, 'bin', self.test_binary)]
xml_content = utils.system_output(' '.join(
['android-sh', '-c', '\"cat',
self.media_profiles_path + '\"']))
root = xml.etree.ElementTree.fromstring(xml_content)
recording_params = Set()
for camcorder_profiles in root.findall('CamcorderProfiles'):
for encoder_profile in camcorder_profiles.findall(
'EncoderProfile'):
video = encoder_profile.find('Video')
recording_params.add(
'%s:%s:%s:%s' %
(camcorder_profiles.get('cameraId'),
video.get('width'), video.get('height'),
video.get('frameRate')))
if recording_params:
cmd.append('--recording_params=' + ','.join(recording_params))
if utils.get_current_board() in self.tablet_board_list:
cmd.append('--gtest_filter=-*SensorOrientationTest/*')
utils.system(' '.join(cmd), timeout=self.timeout)
def run_once(self):
""" Run the regression test and collect the results.
"""
board = utils.get_current_board()
platform = _PLATFORM_MAPPINGS.get(board, board)
# find paths for touchpad tests
root = os.path.join(self.autodir, 'deps', 'touchpad-tests')
framework_dir = os.path.join(root, 'framework')
tests_dir = os.path.join(root, 'tests')
# create test runner
sys.path.append(framework_dir)
sys.path.append(root)
from test_runner import ParallelTestRunner
runner = ParallelTestRunner(tests_dir)
# run all tests for this platform and extract results
results = runner.RunAll('%s*/*' % platform, verbose=True)
# TODO(dennisjeffrey): Remove all uses of self.test_results below,
# including the call to self.write_perf_keyval(), once we're ready to
# switch over completely from perf keyvals to output_perf_value().
self.test_results = {}
for key, value in results.items():
score = value['score']
not_integer = isinstance(score, bool) or not isinstance(score, int)
if not_integer and not isinstance(score, float):
score = 0.0
self.test_results[key.replace('/', '-')] = score
self.output_perf_value(key.replace('/', '-'), score, 'points')
# write converted test results out
if self.test_results:
self.write_perf_keyval(self.test_results)
def run_once(self):
"""Tries to create a socket with every possible combination of
protocol family and socket type.
Fails if it can create a socket for one or more protocol families
not in the baseline.
"""
unexpected_protocol_families = []
# Protocol families currently go up to 40, but this way we make sure
# to catch new families that might get added to the kernel.
for pfn in range(256):
pf_available = self.is_protocol_family_available(pfn)
protocol_family = self.pf_name(pfn)
if pf_available:
# If PF is in baseline, continue.
if protocol_family in self.PF_BASELINE:
continue
# Check the board-specific whitelist.
current_board = utils.get_current_board()
board_pfs = self.PER_BOARD.get(current_board, None)
if not board_pfs or protocol_family not in board_pfs:
unexpected_protocol_families.append(protocol_family)
if len(unexpected_protocol_families) > 0:
failure_string = "Unexpected protocol families available: "
failure_string += ", ".join(unexpected_protocol_families)
logging.error(failure_string)
raise error.TestFail(failure_string)
def run_once(self):
# TODO(scottz): Remove this when crbug.com/220147 is fixed.
dut_board = utils.get_current_board()
if dut_board == 'x86-mario':
raise error.TestNAError('This test is not available on %s' %
dut_board)
with chrome.Chrome() as cr:
self.run_video_tests(cr.browser)
def run_once(self):
"""Runs the video_WebRtcPeerConnectionWithCamera test."""
self.board = utils.get_current_board()
with chrome.Chrome(extra_browser_args=EXTRA_BROWSER_ARGS) as cr:
# Open WebRTC loopback page and start the loopback.
self.start_loopback(cr)
if not self.check_loopback_result():
raise error.TestFail('Failed webrtc camera test')
def run_once(self):
""" Compare expected and actual kernel versions. """
board = utils.get_current_board()
actual = self._actual_kernel(board)
expected = self._expected_kernel(board)
if not actual.startswith(expected):
raise error.TestFail('%s: Expected kernel version %s; Found %s' %
(board, expected, actual))
def skip_devices_to_test(*boards):
"""Devices to skip due to hardware or test compatibility issues.
@param boards: the boards to skip testing.
"""
# TODO(scottz): Remove this when crbug.com/220147 is fixed.
dut_board = utils.get_current_board()
if dut_board in boards:
raise error.TestNAError('This test is not available on %s' % dut_board)
def run_once(self):
boards_to_skip = ['x86-mario', 'x86-zgb']
# TODO(scottz): Remove this when crbug.com/220147 is fixed.
dut_board = utils.get_current_board()
if dut_board in boards_to_skip:
logging.info("Skipping test run on this board.")
return
with chrome.Chrome() as cr:
self.run_video_sanity_test(cr.browser)
def run_once(self):
# TODO(scottz): Remove this when crbug.com/220147 is fixed.
dut_board = utils.get_current_board()
if dut_board == 'x86-mario':
raise error.TestNAError('This test is not available on %s' %
dut_board)
with chrome.Chrome(
extra_browser_args=helper_logger.chrome_vmodule_flag(),
init_network_controller=True) as cr:
self.run_video_tests(cr.browser)
def run_once(self, video_file, arc_mode=False):
"""
Tests whether the requested video is playable
@param video_file: Sample video file to be played in Chrome.
"""
blacklist = [
# (board, arc_mode) # None means don't care
('x86-mario', None),
('x86-zgb', None),
# The result on elm and oak is flaky in arc mode.
# TODO(wuchengli): remove them once crbug.com/679864 is fixed.
('elm', True),
('oak', True)
]
dut_board = utils.get_current_board()
for (blacklist_board, blacklist_arc_mode) in blacklist:
if blacklist_board == dut_board:
if blacklist_arc_mode is None or blacklist_arc_mode == arc_mode:
logging.info("Skipping test run on this board.")
return
break
if arc_mode:
arc_mode_str = arc_common.ARC_MODE_ENABLED
else:
arc_mode_str = arc_common.ARC_MODE_DISABLED
with chrome.Chrome(
extra_browser_args=helper_logger.chrome_vmodule_flag(),
arc_mode=arc_mode_str,
init_network_controller=True) as cr:
shutil.copy2(constants.VIDEO_HTML_FILEPATH, self.bindir)
video_path = os.path.join(constants.CROS_VIDEO_DIR, 'files',
video_file)
shutil.copy2(video_path, self.bindir)
cr.browser.platform.SetHTTPServerDirectories(self.bindir)
tab = cr.browser.tabs.New()
html_fullpath = os.path.join(self.bindir, 'video.html')
url = cr.browser.platform.http_server.UrlOf(html_fullpath)
player = native_html5_player.NativeHtml5Player(
tab,
full_url=url,
video_id='video',
video_src_path=video_file,
event_timeout=120)
player.load_video()
player.play()
player.verify_video_can_play(constants.PLAYBACK_TEST_TIME_S)
def _load_baseline(self, bltype):
"""Load the list of expected files for a given baseline type.
@param bltype the baseline to load.
@returns a set containing the names of the files in the board's
baseline.
"""
# Baseline common to all boards.
blname = 'baseline.' + bltype
blset = self._load_baseline_file(blname)
# Board-specific baseline.
board_blname = 'baseline.%s.%s' % (utils.get_current_board(), bltype)
blset |= self._load_baseline_file(board_blname)
return blset
def get_filter_option(self):
"""Get option of filtering test
"""
blacklist = {
# board: [tests to skip...]
# Kevin doesn't support HW encode for plane sizes not multiple
# of cache line
'kevin': ['CacheLineUnalignedInputTest/*']
}
board = utils.get_current_board()
if board in blacklist:
return ' --gtest_filter=-' + ':'.join(blacklist[board])
return ''
def _can_switch_bitrate():
"""Determine whether the board can switch the bitrate.
The bitrate switch test case is mainly for ARC++. We don't have much control
over some devices that do not run ARC++. Therefore we whitelist the boards
that should pass the bitrate switch test. (crbug.com/890125)
"""
# Most Intel chipsets are able to switch bitrate except these two old
# chipsets, so we blacklist the devices.
intel_blacklist = [
# Rambi Bay Trial
'cranky',
'banjo',
'candy',
'clapper',
'enguarde',
'expresso',
'glimmer',
'gnawty',
'heli',
'hoofer',
'kip',
'kip14',
'ninja',
'orco',
'quawks',
'squawks',
'sumo',
'swanky',
'winky',
# Haswell
'falco',
'leon',
'mccloud',
'monroe',
'panther',
'peppy',
'tricky',
'wolf',
'zako',
]
return (_run_on_intel_cpu()
and utils.get_current_board() not in intel_blacklist)
def run_once(self):
"""Entry point of this test."""
# Flaky on nyan_* boards. http://crbug.com/717275
boards_to_skip = ['nyan_big', 'nyan_kitty', 'nyan_blaze']
if utils.get_current_board() in boards_to_skip:
logging.warning('Skipping test run on this board.')
return
# GPU info collection via devtools SystemInfo.getInfo does not work
# under mus due to differences in how the GPU process is configured.
# http://crbug.com/669965
mus_browser_args = ['--mus', '--gpu-no-complete-info-collection']
logging.info('Testing Chrome --mus startup.')
with chrome.Chrome(auto_login=False,
extra_browser_args=mus_browser_args):
logging.info('Chrome --mus started and loaded OOBE.')
logging.info('Testing Chrome --mus login.')
with chrome.Chrome(extra_browser_args=mus_browser_args):
logging.info('Chrome login with --mus succeeded.')
def run_once(self, capability):
"""Runs the test.
@param capability: Capability required for executing this test.
"""
device_capability.DeviceCapability().ensure_capability(capability)
self.board = utils.get_current_board()
with chrome.Chrome(extra_browser_args=EXTRA_BROWSER_ARGS +\
[helper_logger.chrome_vmodule_flag()],
init_network_controller=True) as cr:
# TODO(keiichiw): vivid should be loaded in self.setup() after
# crbug.com/871185 is fixed
if utils.is_virtual_machine():
try:
utils.run('sudo modprobe vivid n_devs=1 node_types=0x1')
except Exception as e:
raise error.TestFail('Failed to load vivid', e)
self.start_getusermedia(cr)
self.print_perf_results()
def _can_encode_nv12():
"""
Determine whether the board can encode NV12.
NV12 format is a mostly common input format driver supports in video
encoding.
There are devices that cannot encode NV12 input buffer because of chromium
code base or driver issue.
"""
# Although V4L2VEA supports NV12, some devices cannot encode NV12 probably
# due to a driver issue.
nv12_black_list = [
r'^daisy.*',
r'^nyan.*',
r'^peach.*',
r'^veyron.*',
]
board = utils.get_current_board()
for p in nv12_black_list:
if re.match(p, board):
return False
return True
def run_once(self):
# TODO(apronin): crbug.com/618392. This test flakes on these boards.
boards_to_skip = ['tricky', 'peach_pit', 'falco']
board = utils.get_current_board()
if board in boards_to_skip:
logging.info("Skipping test run on %s.", board)
return
listener = session_manager.OwnershipSignalListener(gobject.MainLoop())
listener.listen_for_new_key_and_policy()
# Sign in. Sign out happens automatically when cr goes out of scope.
with chrome.Chrome(clear_enterprise_policy=False) as cr:
listener.wait_for_signals(desc='Owner settings written to disk.')
key = open(constants.OWNER_KEY_FILE, 'rb')
hash = hashlib.md5(key.read())
key.close()
mtime = os.stat(constants.OWNER_KEY_FILE).st_mtime
# Sign in/sign out as a second user.
with chrome.Chrome(clear_enterprise_policy=False,
username=self._TEST_USER,
password=self._TEST_PASS,
gaia_id=self._TEST_GAIAID) as cr:
pass
# Checking mtime to see if key file was touched during second sign in.
if os.stat(constants.OWNER_KEY_FILE).st_mtime > mtime:
raise error.TestFail("Owner key was touched on second login!")
# Sanity check.
key2 = open(constants.OWNER_KEY_FILE, 'rb')
hash2 = hashlib.md5(key2.read())
key2.close()
if hash.hexdigest() != hash2.hexdigest():
raise error.TestFail("Owner key was touched on second login!")
def run_once(self):
boards_to_skip = ['x86-mario']
dut_board = utils.get_current_board()
if dut_board in boards_to_skip:
logging.info("Skipping test run on this board.")
return
# Check for existing cras crashes which might occur during UI bring up.
# TODO: (rohitbm) check if we need to reboot the DUT before the test
# start to verify cras crashes during boot.
existing_crash_reports = self.collect_cras_crash()
if len(existing_crash_reports) == 0:
self._check['crashes_on_boot'] = True
# Capturing cras pid before startig the test.
cras_pid_1 = utils.get_oldest_pid_by_name('/usr/bin/cras')
with chrome.Chrome(init_network_controller=True) as self._cr:
try:
# This will be used on Chrome PFQ since it's using a more recent
# version of Chrome. crbug.com/537655.
self._cr.browser.platform.SetHTTPServerDirectories(self.bindir)
except:
# This will be used on ChromeOS CQ since Chrome hasn't uprev'ed
# yet. crbug.com/538140.
self._cr.browser.SetHTTPServerDirectories(self.bindir)
for test_file in self._audio:
url = _DOWNLOAD_BASE + 'audio_test/' + test_file
self.push_new_stream(self._cr.browser.tabs.New(), url)
# Capturing cras pid before opening a new set of audio streams.
cras_pid_2 = utils.get_oldest_pid_by_name('/usr/bin/cras')
for test_file in self._video:
url = _DOWNLOAD_BASE + 'traffic/' + test_file
self.push_new_stream(self._cr.browser.tabs.New(), url)
# Let's play audio for sometime to ensure that
# long playback is good.
time.sleep(10)
total_tests = len(self._audio) + len(self._video)
active_streams = cras_utils.get_active_stream_count()
logging.debug(
'Number of active streams after opening all tabs: %d.',
active_streams)
if active_streams >= total_tests:
self._check['stream_activation'] = True
# Capturing cras pid after opening all audio/video streams.
cras_pid_3 = utils.get_oldest_pid_by_name('/usr/bin/cras')
# Close all open audio streams.
while total_tests > 0:
self._cr.browser.tabs[total_tests].Close()
total_tests -= 1
time.sleep(1)
active_streams = cras_utils.get_active_stream_count()
logging.debug(
'Number of active streams after closing all tabs: %d.',
active_streams)
# Capturing cras pid after closing all audio/stream streams.
cras_pid_4 = utils.get_oldest_pid_by_name('/usr/bin/cras')
if cras_pid_1 == cras_pid_2 == cras_pid_3 == cras_pid_4:
self._check['cras_status'] = True
new_crash_reports = self.collect_cras_crash()
new_reports = list(
set(new_crash_reports) - set(existing_crash_reports))
if len(new_reports) == 0:
self._check['crashes_at_end'] = True
err_msg = ''
if self._check.values().count(False) > 0:
if not self._check['crashes_on_boot']:
err_msg = ('1. Found cras crashes on boot: %s.\n' %
existing_crash_reports)
if not self._check['stream_activation']:
err_msg += ('2. CRAS stream count is not matching with '
'number of streams.\n')
if not self._check['cras_status']:
err_msg += ('CRAS PID changed during the test. CRAS might be '
'crashing while adding/removing streams.\n')
if not self._check['crashes_at_end']:
err_msg += ('Found cras crashes at the end of the test : %s.' %
new_reports)
raise error.TestError(err_msg)
def run_once(self,
source_path,
codec,
resolution,
host,
args,
collect_only=False):
board = utils.get_current_board()
file_utils.make_leaf_dir(constants.TEST_DIR)
with chrome.Chrome(
extension_paths=[cros_constants.MULTIMEDIA_TEST_EXTENSION],
autotest_ext=True) as cr:
cr.browser.platform.SetHTTPServerDirectories(self.bindir)
html_fullpath = os.path.join(self.bindir, 'video.html')
player = native_html5_player.NativeHtml5Player(
tab=cr.browser.tabs[0],
full_url=cr.browser.platform.http_server.UrlOf(html_fullpath),
video_id='video',
video_src_path=source_path,
event_timeout=120)
chameleon_board = chameleon.create_chameleon_board(
host.hostname, args)
display_facade = local_facade_factory.LocalFacadeFactory(
cr).create_display_facade()
finder = chameleon_port_finder.ChameleonVideoInputFinder(
chameleon_board, display_facade)
capturer = chameleon_video_capturer.ChameleonVideoCapturer(
finder.find_port(interface='hdmi'), display_facade)
with capturer:
player.load_video()
player.verify_video_can_play()
display_facade.move_to_display(
display_facade.get_first_external_display_index())
display_facade.set_fullscreen(True)
# HACK: Unset and reset fullscreen. There is a bug in Chrome
# that fails to move the window to a correct position.
# Resetting fullscren helps, check http://crbug.com/574284
display_facade.set_fullscreen(False)
display_facade.set_fullscreen(True)
time.sleep(5)
box = (0, 0, constants.DESIRED_WIDTH, constants.DESIRED_HEIGHT)
#TODO: mussa, Revisit once crbug/580736 is fixed
for n in xrange(constants.NUM_CAPTURE_TRIES):
logging.debug('Trying to capture frames. TRY #%d', n + 1)
raw_test_checksums = capturer.capture_only(
player, max_frame_count=constants.FCOUNT, box=box)
raw_test_checksums = [
tuple(checksum) for checksum in raw_test_checksums
]
overreach_counts = self.overreach_frame_counts(
raw_test_checksums, constants.MAX_FRAME_REPEAT_COUNT)
if not overreach_counts: # no checksums exceeded threshold
break
player.pause()
player.seek_to(datetime.timedelta(seconds=0))
else:
msg = ('Framecount overreach detected even after %d '
'tries. Checksums: %s' %
(constants.NUM_CAPTURE_TRIES, overreach_counts))
raise error.TestFail(msg)
# produces unique checksums mapped to their occur. indices
test_checksum_indices = frame_checksum_utils.checksum_indices(
raw_test_checksums)
test_checksums = test_checksum_indices.keys()
test_indices = test_checksum_indices.values()
golden_checksums_filepath = os.path.join(
constants.TEST_DIR, constants.GOLDEN_CHECKSUMS_FILENAME)
if collect_only:
capturer.write_images(test_indices, constants.TEST_DIR,
constants.IMAGE_FORMAT)
logging.debug("Write golden checksum file to %s",
golden_checksums_filepath)
with open(golden_checksums_filepath, "w+") as f:
for checksum in test_checksums:
f.write(' '.join([str(i)
for i in checksum]) + '\n')
return
golden_checksums_remote_filepath = os.path.join(
constants.GOLDEN_CHECKSUM_REMOTE_BASE_DIR, board,
codec + '_' + resolution,
constants.GOLDEN_CHECKSUMS_FILENAME)
file_utils.download_file(golden_checksums_remote_filepath,
golden_checksums_filepath)
golden_checksums = self.read_checksum_file(
golden_checksums_filepath)
golden_checksum_count = len(golden_checksums)
test_checksum_count = len(test_checksums)
eps = constants.MAX_DIFF_TOTAL_FCOUNT
if golden_checksum_count - test_checksum_count > eps:
msg = ('Expecting about %d checksums, received %d. '
'Allowed delta is %d') % (golden_checksum_count,
test_checksum_count, eps)
raise error.TestFail(msg)
# Some frames might be missing during either golden frame
# collection or during a test run. Using LCS ensures we
# ignore a few missing frames while comparing test vs golden
lcs_len = sequence_utils.lcs_length(golden_checksums,
test_checksums)
missing_frames_count = len(golden_checksums) - lcs_len
unknown_frames_count = len(test_checksums) - lcs_len
msg = ('# of matching frames : %d. # of missing frames : %d. '
'# of unknown test frames : %d. Max allowed # of '
'missing frames : %d. # of golden frames : %d. # of '
'test_checksums : %d' %
(lcs_len, missing_frames_count,
unknown_frames_count, constants.MAX_NONMATCHING_FCOUNT,
len(golden_checksums), len(test_checksums)))
logging.debug(msg)
if (missing_frames_count + unknown_frames_count >
constants.MAX_NONMATCHING_FCOUNT):
unknown_frames = set(test_checksums) - set(
golden_checksums)
store_indices = [
test_checksum_indices[c] for c in unknown_frames
]
paths = capturer.write_images(store_indices,
constants.TEST_DIR,
constants.IMAGE_FORMAT)
path_publish = publisher.ImageDiffPublisher(
self.resultsdir)
path_publish.publish_paths(paths, self.tagged_testname)
raise error.TestFail("Too many non-matching frames")
def get_current_board(self):
"""Returns the current device board name."""
return utils.get_current_board()
def run_once(self,
cmd_timeout=600,
camera_hals=None,
options=None,
capability=None,
test_config=None):
"""
Entry point of this test.
@param cmd_timeout: Seconds. Timeout for running the test command.
@param camera_hals: The camera HALs to be tested. e.g. ['usb.so']
@param options: Option strings passed to test command. e.g. ['--v=1']
@param capability: Capability required for executing this test.
"""
if options is None:
options = []
if test_config is None:
test_config = {}
if capability:
device_capability.DeviceCapability().ensure_capability(capability)
self.job.install_pkg(self.dep, 'dep', self.dep_dir)
camera_hal_paths = camera_utils.get_camera_hal_paths_for_test()
if camera_hals is not None:
name_map = dict(
(os.path.basename(path), path) for path in camera_hal_paths)
camera_hal_paths = []
for name in camera_hals:
path = name_map.get(name)
if path is None:
msg = 'HAL %r is not available for test' % name
raise error.TestNAError(msg)
camera_hal_paths.append(path)
binary_path = os.path.join(self.dep_dir, 'bin', self.test_binary)
with service_stopper.ServiceStopper([self.cros_camera_service]), \
self.set_test_config(test_config):
has_facing_option = False
cmd = [binary_path]
for option in options:
if 'gtest_filter' in option:
filters = option.split('=')[1]
if 'Camera3DeviceTest' in filters.split('-')[0]:
if utils.get_current_board() in self.tablet_board_list:
option += (':' if '-' in filters else '-')
option += '*SensorOrientationTest/*'
if any(name in filters.split('-')[0]
for name in ('Camera3ModuleFixture',
'Camera3RecordingFixture')):
cmd.append(self.get_recording_params())
elif 'camera_facing' in option:
has_facing_option = True
cmd.append(option)
if has_facing_option:
utils.system(cmd, timeout=cmd_timeout)
else:
for camera_hal_path in camera_hal_paths:
logging.info('Run test with %r', camera_hal_path)
cmd.append('--camera_hal_path=%s' % camera_hal_path)
utils.system(cmd, timeout=cmd_timeout)
cmd.pop()
def run_once(self):
boards_to_skip = ['x86-mario']
dut_board = utils.get_current_board()
if dut_board in boards_to_skip:
logging.info("Skipping test run on this board.")
return
# Check for existing cras crashes which might occur during UI bring up.
# TODO: (rohitbm) check if we need to reboot the DUT before the test
# start to verify cras crashes during boot.
existing_crash_reports = self.collect_cras_crash()
if len(existing_crash_reports) == 0:
self._check['crashes_on_boot'] = True
# Capturing cras pid before startig the test.
cras_pid_1 = utils.get_oldest_pid_by_name('/usr/bin/cras')
with chrome.Chrome(init_network_controller=True) as self._cr:
# Push the 1st stream
self.push_new_stream(self._cr.browser.tabs.New())
# Capturing cras pid before opening a new set of audio streams.
cras_pid_2 = utils.get_oldest_pid_by_name('/usr/bin/cras')
# Push the 2nd stream
self.push_new_stream(self._cr.browser.tabs.New())
# Let's play audio for sometime to ensure that
# long playback is good.
time.sleep(10)
total_tests = 2
active_streams = cras_utils.get_active_stream_count()
logging.debug(
'Number of active streams after opening all tabs: %d.',
active_streams)
if active_streams >= total_tests:
self._check['stream_activation'] = True
# Capturing cras pid after opening all audio/video streams.
cras_pid_3 = utils.get_oldest_pid_by_name('/usr/bin/cras')
# Close all open audio streams.
while total_tests > 0:
self._cr.browser.tabs[total_tests].Close()
total_tests -= 1
time.sleep(1)
active_streams = cras_utils.get_active_stream_count()
logging.debug(
'Number of active streams after closing all tabs: %d.',
active_streams)
# Capturing cras pid after closing all audio/stream streams.
cras_pid_4 = utils.get_oldest_pid_by_name('/usr/bin/cras')
if cras_pid_1 == cras_pid_2 == cras_pid_3 == cras_pid_4:
self._check['cras_status'] = True
new_crash_reports = self.collect_cras_crash()
new_reports = list(
set(new_crash_reports) - set(existing_crash_reports))
if len(new_reports) == 0:
self._check['crashes_at_end'] = True
err_msg = ''
if self._check.values().count(False) > 0:
if not self._check['crashes_on_boot']:
err_msg = ('1. Found cras crashes on boot: %s.\n' %
existing_crash_reports)
if not self._check['stream_activation']:
err_msg += ('2. CRAS stream count is not matching with '
'number of streams.\n')
if not self._check['cras_status']:
err_msg += ('CRAS PID changed during the test. CRAS might be '
'crashing while adding/removing streams.\n')
if not self._check['crashes_at_end']:
err_msg += ('Found cras crashes at the end of the test : %s.' %
new_reports)
raise error.TestError(err_msg)
def compare_extensions(self):
"""Compare installed extensions to the expected set.
Find the set of expected IDs.
Find the set of observed IDs.
Do set comparison to find the unexpected, and the expected/missing.
"""
test_fail = False
combined_baseline = (self._bundled_crx_baseline +
self._component_extension_baseline)
# Filter out any baseline entries that don't apply to this board.
# If there is no 'boards' limiter on a given record, the record applies.
# If there IS a 'boards' limiter, check that it applies.
board = utils.get_current_board()
combined_baseline = [
x for x in combined_baseline
if ((not 'boards' in x) or ('boards' in x and board in x['boards'])
)
]
observed_extensions = self._get_extensions_info()
observed_ids = set([x['id'] for x in observed_extensions])
expected_ids = set([x['id'] for x in combined_baseline])
missing_ids = expected_ids - observed_ids
missing_names = [
'%s (%s)' % (x['name'], x['id']) for x in combined_baseline
if x['id'] in missing_ids
]
unexpected_ids = observed_ids - expected_ids
unexpected_names = [
'%s (%s)' % (x['name'], x['id']) for x in observed_extensions
if x['id'] in unexpected_ids
]
good_ids = expected_ids.intersection(observed_ids)
if missing_names:
logging.error('Missing: %s', '; '.join(missing_names))
test_fail = True
if unexpected_names:
logging.error('Unexpected: %s', '; '.join(unexpected_names))
test_fail = True
# For those IDs in both the expected-and-observed, ie, "good":
# Compare sets of expected-vs-actual API permissions, report diffs.
# Do same for host permissions.
for good_id in good_ids:
baseline = [x for x in combined_baseline if x['id'] == good_id][0]
actual = [x for x in observed_extensions if x['id'] == good_id][0]
# Check the API permissions.
baseline_apis = set(baseline['apiPermissions'])
actual_apis = set(actual['apiPermissions'])
missing_apis = baseline_apis - actual_apis
unexpected_apis = actual_apis - baseline_apis
if missing_apis or unexpected_apis:
test_fail = True
self._report_attribute_diffs(missing_apis, unexpected_apis,
actual)
# Check the host permissions.
baseline_hosts = set(baseline['effectiveHostPermissions'])
actual_hosts = set(actual['effectiveHostPermissions'])
missing_hosts = baseline_hosts - actual_hosts
unexpected_hosts = actual_hosts - baseline_hosts
if missing_hosts or unexpected_hosts:
test_fail = True
self._report_attribute_diffs(missing_hosts, unexpected_hosts,
actual)
if test_fail:
# TODO(jorgelo): make this fail again, see crbug.com/343271.
raise error.TestWarn('Baseline mismatch, see error log.')
