def FetchBinaryDepdencies(platform, client_configs,
fetch_reference_chrome_binary):
""" Fetch all binary dependenencies for the given |platform|.
Note: we don't fetch browser binaries by default because the size of the
binary is about 2Gb, and it requires cloud storage permission to
chrome-telemetry bucket.
Args:
platform: an instance of telemetry.core.platform
client_configs: A list of paths (string) to dependencies json files.
fetch_reference_chrome_binary: whether to fetch reference chrome binary for
the given platform.
"""
configs = [
dependency_manager.BaseConfig(TELEMETRY_PROJECT_CONFIG),
dependency_manager.BaseConfig(BATTOR_BINARY_CONFIG)
]
dep_manager = dependency_manager.DependencyManager(configs)
target_platform = '%s_%s' % (platform.GetOSName(), platform.GetArchName())
host_platform = None
dep_manager.PrefetchPaths(target_platform)
if platform.GetOSName() == 'android':
host_platform = '%s_%s' % (platform_module.GetHostPlatform().GetOSName(
), platform_module.GetHostPlatform().GetArchName())
dep_manager.PrefetchPaths(host_platform)
if fetch_reference_chrome_binary:
_FetchReferenceBrowserBinary(platform)
# For now, handle client config separately because the BUILD.gn & .isolate of
# telemetry tests in chromium src failed to include the files specified in its
# client config.
# (https://github.com/catapult-project/catapult/issues/2192)
# For now this is ok because the client configs usually don't include cloud
# storage infos.
# TODO(nednguyen): remove the logic of swallowing exception once the issue is
# fixed on Chromium side.
if client_configs:
manager = dependency_manager.DependencyManager(
list(dependency_manager.BaseConfig(c) for c in client_configs))
try:
manager.PrefetchPaths(target_platform)
if host_platform is not None:
manager.PrefetchPaths(host_platform)
except dependency_manager.NoPathFoundError as e:
logging.error('Error when trying to prefetch paths for %s: %s',
target_platform, e.message)
def _InitializeRecursive(self, configs=None, config_files=None):
# This recurses through configs to create temporary files for each and
# take advantage of context managers to appropriately close those files.
# TODO(jbudorick): Remove this recursion if/when dependency_manager
# supports loading configurations directly from a dict.
if configs:
with tempfile.NamedTemporaryFile(delete=False) as next_config_file:
try:
next_config_file.write(json.dumps(configs[0]))
next_config_file.close()
self._InitializeRecursive(
configs=configs[1:],
config_files=[next_config_file.name] +
(config_files or []))
finally:
if os.path.exists(next_config_file.name):
os.remove(next_config_file.name)
else:
config_files = config_files or []
if 'DEVIL_ENV_CONFIG' in os.environ:
config_files.append(os.environ.get('DEVIL_ENV_CONFIG'))
config_files.append(_DEVIL_DEFAULT_CONFIG)
self._dm = dependency_manager.DependencyManager(
[dependency_manager.BaseConfig(c) for c in config_files])
def testFetchPathError(self, cs_path_mock):
dep_manager = dependency_manager.DependencyManager([])
self.assertFalse(cs_path_mock.call_args)
cs_path_mock.return_value = None
dep_manager._lookup_dict = {
'dep': {
'platform': self.dep_info,
'plat1': mock.MagicMock()
},
'dep2': {
'plat2': mock.MagicMock()
}
}
# Non-empty lookup dict that contains the dependency we're looking for.
# Local path doesn't exist, and cloud_storage path wasn't successfully
# found.
self.assertRaises(exceptions.NoPathFoundError, dep_manager.FetchPath,
'dep', 'platform')
cs_path_mock.side_effect = cloud_storage.CredentialsError
self.assertRaises(cloud_storage.CredentialsError,
dep_manager.FetchPath, 'dep', 'platform')
cs_path_mock.side_effect = cloud_storage.CloudStorageError
self.assertRaises(cloud_storage.CloudStorageError,
dep_manager.FetchPath, 'dep', 'platform')
cs_path_mock.side_effect = cloud_storage.PermissionError
self.assertRaises(cloud_storage.PermissionError, dep_manager.FetchPath,
'dep', 'platform')
def __init__(self, config_files):
if not config_files or not isinstance(config_files, list):
raise ValueError(
'Must supply a list of config files to the BinaryManager')
configs = [
dependency_manager.BaseConfig(config) for config in config_files
]
self._dependency_manager = dependency_manager.DependencyManager(
configs)
def testLocalPathNoDependency(self):
# Non-empty lookup dict that doesn't contain the dependency we're looking
# for.
dep_manager = dependency_manager.DependencyManager([])
dep_manager._lookup_dict = {
'dep1': mock.MagicMock(),
'dep2': mock.MagicMock()
}
with self.assertRaises(exceptions.NoPathFoundError):
dep_manager.LocalPath('dep', 'plat')
def __init__(self,
target_platform,
android_device=None,
battor_path=None,
battor_map_file=None,
battor_map=None,
serial_log_bucket=None,
autoflash=True):
"""Constructor.
Args:
target_platform: Platform BattOr is attached to.
android_device: Serial number of Android device.
battor_path: Path to BattOr device.
battor_map_file: File giving map of [device serial: BattOr path]
battor_map: Map of [device serial: BattOr path]
serial_log_bucket: The cloud storage bucket to which BattOr agent serial
logs are uploaded on failure.
Attributes:
_battor_path: Path to BattOr. Typically similar to /tty/USB0.
_battor_agent_binary: Path to the BattOr agent binary used to communicate
with the BattOr.
_tracing: A bool saying if tracing has been started.
_battor_shell: A subprocess running the battor_agent_binary
_trace_results_path: Path to BattOr trace results file.
_serial_log_bucket: Cloud storage bucket to which BattOr agent serial logs
are uploaded on failure.
_serial_log_file: Temp file for the BattOr agent serial log.
"""
self._battor_path = self._GetBattOrPath(target_platform,
android_device, battor_path,
battor_map_file, battor_map)
config = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'battor_binary_dependencies.json')
self._dm = dependency_manager.DependencyManager(
[dependency_manager.BaseConfig(config)])
self._battor_agent_binary = self._dm.FetchPath(
'battor_agent_binary',
'%s_%s' % (sys.platform, platform.machine()))
self._autoflash = autoflash
self._serial_log_bucket = serial_log_bucket
self._tracing = False
self._battor_shell = None
self._trace_results_path = None
self._start_tracing_time = None
self._stop_tracing_time = None
self._trace_results = None
self._serial_log_file = None
self._target_platform = target_platform
self._git_hash = None
atexit.register(self.KillBattOrShell)
def __init__(self, target_platform, android_device=None, battor_path=None,
battor_map_file=None, battor_map=None, serial_log_bucket=None):
"""Constructor.
Args:
target_platform: Platform BattOr is attached to.
android_device: Serial number of Android device.
battor_path: Path to BattOr device.
battor_map_file: File giving map of [device serial: BattOr path]
battor_map: Map of [device serial: BattOr path]
serial_log_bucket: The cloud storage bucket to which BattOr agent serial
logs are uploaded on failure.
Attributes:
_battor_path: Path to BattOr. Typically similar to /tty/USB0.
_battor_agent_binary: Path to the BattOr agent binary used to communicate
with the BattOr.
_tracing: A bool saying if tracing has been started.
_battor_shell: A subprocess running the battor_agent_binary
_trace_results_path: Path to BattOr trace results file.
_serial_log_bucket: Cloud storage bucket to which BattOr agent serial logs
are uploaded on failure.
_serial_log_file: Temp file for the BattOr agent serial log.
"""
self._battor_path = self._GetBattOrPath(target_platform, android_device,
battor_path, battor_map_file, battor_map)
config = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
'battor_binary_dependencies.json')
dm = dependency_manager.DependencyManager(
[dependency_manager.BaseConfig(config)])
self._battor_agent_binary = dm.FetchPath(
'battor_agent_binary', '%s_%s' % (sys.platform, platform.machine()))
# Remove this when we have windows avrdude binary uploaded.
# https://github.com/catapult-project/catapult/issues/2972
if target_platform in self._SUPPORTED_AUTOFLASHING_PLATFORMS:
self._avrdude_binary = dm.FetchPath(
'avrdude_binary', '%s_%s' % (sys.platform, platform.machine()))
self._serial_log_bucket = serial_log_bucket
self._tracing = False
self._battor_shell = None
self._trace_results_path = None
self._start_tracing_time = None
self._stop_tracing_time = None
self._trace_results = None
self._serial_log_file = None
self._target_platform = target_platform
atexit.register(self.KillBattOrShell)
def testLocalPathMissingPaths(self):
# Non-empty lookup dict that contains the dependency we're looking for.
# Local path is found but doesn't exist.
dep_manager = dependency_manager.DependencyManager([])
dep_manager._lookup_dict = {
'dependency': {
'platform': self.dep_info
},
'dep1': mock.MagicMock(),
'dep2': mock.MagicMock()
}
self.assertRaises(exceptions.NoPathFoundError, dep_manager.LocalPath,
'dependency', 'platform')
def testLocalPathExists(self):
# Non-empty lookup dict that contains the dependency we're looking for.
# Local path exists.
dep_manager = dependency_manager.DependencyManager([])
dep_manager._lookup_dict = {
'dependency': {
'platform': self.dep_info
},
'dep1': mock.MagicMock(),
'dep2': mock.MagicMock()
}
self.fs.CreateFile('path1')
found_path = dep_manager.LocalPath('dependency', 'platform')
self.assertEqual('path1', found_path)
def testGetDependencyInfo(self):
dep_manager = dependency_manager.DependencyManager([])
self.assertFalse(dep_manager._lookup_dict)
# No dependencies in the dependency manager.
self.assertEqual(
None, dep_manager._GetDependencyInfo('missing_dep',
'missing_plat'))
dep_manager._lookup_dict = {
'dep1': {
'plat1': 'dep_info11',
'plat2': 'dep_info12',
'plat3': 'dep_info13'
},
'dep2': {
'plat1': 'dep_info11',
'plat2': 'dep_info21',
'plat3': 'dep_info23',
'default': 'dep_info2d'
},
'dep3': {
'plat1': 'dep_info31',
'plat2': 'dep_info32',
'default': 'dep_info3d'
}
}
# Dependency not in the dependency manager.
self.assertEqual(
None, dep_manager._GetDependencyInfo('missing_dep',
'missing_plat'))
# Dependency in the dependency manager, but not the platform. No default.
self.assertEqual(
None, dep_manager._GetDependencyInfo('dep1', 'missing_plat'))
# Dependency in the dependency manager, but not the platform, but a default
# exists.
self.assertEqual(
'dep_info2d',
dep_manager._GetDependencyInfo('dep2', 'missing_plat'))
# Dependency and platform in the dependency manager. A default exists.
self.assertEqual('dep_info23',
dep_manager._GetDependencyInfo('dep2', 'plat3'))
# Dependency and platform in the dependency manager. No default exists.
self.assertEqual('dep_info12',
dep_manager._GetDependencyInfo('dep1', 'plat2'))
def testLocalPathNoPaths(self):
# Non-empty lookup dict that contains the dependency we're looking for.
# Local path isn't found.
dep_manager = dependency_manager.DependencyManager([])
dep_info = dependency_manager.DependencyInfo(
'dep',
'platform',
'config_file',
cloud_storage_info=self.cloud_storage_info)
dep_manager._lookup_dict = {
'dependency': {
'platform': dep_info
},
'dep1': mock.MagicMock(),
'dep2': mock.MagicMock()
}
self.assertRaises(exceptions.NoPathFoundError, dep_manager.LocalPath,
'dependency', 'platform')
def testFetchPathUnititializedDependency(self, cs_path_mock):
dep_manager = dependency_manager.DependencyManager([])
self.assertFalse(cs_path_mock.call_args)
cs_path = 'cs_path'
cs_path_mock.return_value = cs_path
# Empty lookup_dict
with self.assertRaises(exceptions.NoPathFoundError):
dep_manager.FetchPath('dep', 'plat_arch_x86')
# Non-empty lookup dict that doesn't contain the dependency we're looking
# for.
dep_manager._lookup_dict = {
'dep1': mock.MagicMock(),
'dep2': mock.MagicMock()
}
with self.assertRaises(exceptions.NoPathFoundError):
dep_manager.FetchPath('dep', 'plat_arch_x86')
def __init__(self,
target_platform,
android_device=None,
battor_path=None,
battor_map_file=None,
battor_map=None):
"""Constructor.
Args:
target_platform: Platform BattOr is attached to.
android_device: Serial number of Android device.
battor_path: Path to BattOr device.
battor_map_file: File giving map of [device serial: BattOr path]
battor_map: Map of [device serial: BattOr path]
Attributes:
_battor_path: Path to BattOr. Typically similar to /tty/USB0.
_battor_agent_binary: Path to the BattOr agent binary used to communicate
with the BattOr.
_tracing: A bool saying if tracing has been started.
_battor_shell: A subprocess running the bator_agent_binary
_trace_results_path: Path to BattOr trace results file.
"""
self._battor_path = self._GetBattorPath(target_platform,
android_device, battor_path,
battor_map_file, battor_map)
config = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'battor_binary_dependencies.json')
dm = dependency_manager.DependencyManager(
[dependency_manager.BaseConfig(config)])
self._battor_agent_binary = dm.FetchPath(
'battor_agent_binary',
'%s_%s' % (sys.platform, platform.machine()))
self._tracing = False
self._battor_shell = None
self._trace_results_path = None
self._start_tracing_time = None
self._stop_tracing_time = None
self._trace_results = None
def testFetchPathLocalFile(self, cs_path_mock, dep_info_mock, path_mock):
dep_manager = dependency_manager.DependencyManager([])
self.assertFalse(cs_path_mock.call_args)
cs_path = 'cs_path'
dep_info = self.dep_info
cs_path_mock.return_value = cs_path
# The DependencyInfo returned should be passed through to LocalPath.
dep_info_mock.return_value = dep_info
# Non-empty lookup dict that contains the dependency we're looking for.
# Local path exists.
dep_manager._lookup_dict = {
'dep': {
'platform': self.dep_info
},
'dep2': mock.MagicMock()
}
self.fs.CreateFile('path1')
found_path = dep_manager.FetchPath('dep', 'platform')
self.assertEqual('path1', found_path)
self.assertFalse(cs_path_mock.call_args)
def testFetchPathRemoteFile(self, cs_path_mock):
dep_manager = dependency_manager.DependencyManager([])
self.assertFalse(cs_path_mock.call_args)
cs_path = 'cs_path'
def FakeCSPath():
self.fs.CreateFile(cs_path)
return cs_path
cs_path_mock.side_effect = FakeCSPath
# Non-empty lookup dict that contains the dependency we're looking for.
# Local path doesn't exist, but cloud_storage_path is downloaded.
dep_manager._lookup_dict = {
'dep': {
'platform': self.dep_info,
'plat1': mock.MagicMock()
},
'dep2': {
'plat2': mock.MagicMock()
}
}
found_path = dep_manager.FetchPath('dep', 'platform')
self.assertEqual(cs_path, found_path)
def testFollowupUpdateDependenciesNoOverlap(self):
dep_manager = dependency_manager.DependencyManager([])
dep = 'dependency'
dep1 = 'dependency1'
dep2 = 'dependency2'
dep3 = 'dependency3'
plat1 = 'platform1'
plat2 = 'platform2'
plat3 = 'platform3'
dep_info_a = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep_info_a.dependency = dep1
dep_info_a.platform = plat1
dep_info_b = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep_info_b.dependency = dep1
dep_info_b.platform = plat2
dep_info_c = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep_info_c.dependency = dep
dep_info_c.platform = plat1
start_lookup_dict = {
dep: {
plat1: dep_info_a,
plat2: dep_info_b
},
dep1: {
plat1: dep_info_c
}
}
base_config_mock = mock.MagicMock(spec=dependency_manager.BaseConfig)
# Empty BaseConfig.
dep_manager._lookup_dict = start_lookup_dict.copy()
base_config_mock.IterDependencyInfo.return_value = iter([])
dep_manager._UpdateDependencies(base_config_mock)
self.assertEqual(start_lookup_dict, dep_manager._lookup_dict)
# One dependency/platform in a BaseConfig.
dep_manager._lookup_dict = start_lookup_dict.copy()
dep_info = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep_info.dependency = dep3
dep_info.platform = plat1
base_config_mock.IterDependencyInfo.return_value = iter([dep_info])
expected_lookup_dict = {
dep: {
plat1: dep_info_a,
plat2: dep_info_b
},
dep1: {
plat1: dep_info_c
},
dep3: {
plat3: dep_info
}
}
dep_manager._UpdateDependencies(base_config_mock)
self.assertItemsEqual(expected_lookup_dict, dep_manager._lookup_dict)
self.assertFalse(dep_info.Update.called)
self.assertFalse(dep_info_a.Update.called)
self.assertFalse(dep_info_b.Update.called)
self.assertFalse(dep_info_c.Update.called)
# One dependency multiple platforms in a BaseConfig.
dep_manager._lookup_dict = start_lookup_dict.copy()
dep_info1 = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep_info1.dependency = dep2
dep_info1.platform = plat1
dep_info2 = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep_info2.dependency = dep2
dep_info2.platform = plat2
base_config_mock.IterDependencyInfo.return_value = iter(
[dep_info1, dep_info2])
expected_lookup_dict = {
dep: {
plat1: dep_info_a,
plat2: dep_info_b
},
dep1: {
plat1: dep_info_c
},
dep2: {
plat1: dep_info1,
plat2: dep_info2
}
}
dep_manager._UpdateDependencies(base_config_mock)
self.assertEqual(expected_lookup_dict, dep_manager._lookup_dict)
self.assertFalse(dep_info1.Update.called)
self.assertFalse(dep_info2.Update.called)
self.assertFalse(dep_info_a.Update.called)
self.assertFalse(dep_info_b.Update.called)
self.assertFalse(dep_info_c.Update.called)
# Multiple dependencies, multiple platforms in a BaseConfig.
dep_manager._lookup_dict = start_lookup_dict.copy()
dep1 = 'dependency1'
plat1 = 'platform1'
plat2 = 'platform2'
dep_info1 = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep_info1.dependency = dep2
dep_info1.platform = plat1
dep_info2 = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep_info2.dependency = dep2
dep_info2.platform = plat2
dep_info3 = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep_info3.dependency = dep3
dep_info3.platform = plat2
base_config_mock.IterDependencyInfo.return_value = iter(
[dep_info1, dep_info2, dep_info3])
expected_lookup_dict = {
dep: {
plat1: dep_info_a,
plat2: dep_info_b
},
dep1: {
plat1: dep_info_c
},
dep2: {
plat1: dep_info1,
plat2: dep_info2
},
dep3: {
plat2: dep_info3
}
}
dep_manager._UpdateDependencies(base_config_mock)
self.assertEqual(expected_lookup_dict, dep_manager._lookup_dict)
self.assertFalse(dep_info1.Update.called)
self.assertFalse(dep_info2.Update.called)
self.assertFalse(dep_info3.Update.called)
self.assertFalse(dep_info_a.Update.called)
self.assertFalse(dep_info_b.Update.called)
self.assertFalse(dep_info_c.Update.called)
# Ensure the testing data wasn't corrupted.
self.assertEqual(start_lookup_dict, {
dep: {
plat1: dep_info_a,
plat2: dep_info_b
},
dep1: {
plat1: dep_info_c
}
})
def testFollowupUpdateDependenciesWithCollisions(self):
dep_manager = dependency_manager.DependencyManager([])
dep = 'dependency'
dep1 = 'dependency1'
dep2 = 'dependency2'
plat1 = 'platform1'
plat2 = 'platform2'
dep_info_a = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep_info_a.dependency = dep1
dep_info_a.platform = plat1
dep_info_b = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep_info_b.dependency = dep1
dep_info_b.platform = plat2
dep_info_c = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep_info_c.dependency = dep
dep_info_c.platform = plat1
start_lookup_dict = {
dep: {
plat1: dep_info_a,
plat2: dep_info_b
},
dep1: {
plat1: dep_info_c
}
}
base_config_mock = mock.MagicMock(spec=dependency_manager.BaseConfig)
# One dependency/platform.
dep_manager._lookup_dict = start_lookup_dict.copy()
dep_info = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep_info.dependency = dep
dep_info.platform = plat1
base_config_mock.IterDependencyInfo.return_value = iter([dep_info])
expected_lookup_dict = {
dep: {
plat1: dep_info_a,
plat2: dep_info_b
},
dep1: {
plat1: dep_info_c
}
}
dep_manager._UpdateDependencies(base_config_mock)
self.assertItemsEqual(expected_lookup_dict, dep_manager._lookup_dict)
dep_info_a.Update.assert_called_once_with(dep_info)
self.assertFalse(dep_info.Update.called)
self.assertFalse(dep_info_b.Update.called)
self.assertFalse(dep_info_c.Update.called)
dep_info_a.reset_mock()
dep_info_b.reset_mock()
dep_info_c.reset_mock()
# One dependency multiple platforms in a BaseConfig.
dep_manager._lookup_dict = start_lookup_dict.copy()
dep_info1 = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep_info1.dependency = dep1
dep_info1.platform = plat1
dep_info2 = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep_info2.dependency = dep2
dep_info2.platform = plat2
base_config_mock.IterDependencyInfo.return_value = iter(
[dep_info1, dep_info2])
expected_lookup_dict = {
dep: {
plat1: dep_info_a,
plat2: dep_info_b
},
dep1: {
plat1: dep_info_c
},
dep2: {
plat2: dep_info2
}
}
dep_manager._UpdateDependencies(base_config_mock)
self.assertEqual(expected_lookup_dict, dep_manager._lookup_dict)
self.assertFalse(dep_info1.Update.called)
self.assertFalse(dep_info2.Update.called)
self.assertFalse(dep_info_a.Update.called)
self.assertFalse(dep_info_b.Update.called)
dep_info_c.Update.assert_called_once_with(dep_info1)
dep_info_a.reset_mock()
dep_info_b.reset_mock()
dep_info_c.reset_mock()
# Multiple dependencies, multiple platforms in a BaseConfig.
dep_manager._lookup_dict = start_lookup_dict.copy()
dep1 = 'dependency1'
plat1 = 'platform1'
plat2 = 'platform2'
dep_info1 = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep_info1.dependency = dep
dep_info1.platform = plat1
dep_info2 = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep_info2.dependency = dep1
dep_info2.platform = plat1
dep_info3 = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep_info3.dependency = dep2
dep_info3.platform = plat2
base_config_mock.IterDependencyInfo.return_value = iter(
[dep_info1, dep_info2, dep_info3])
expected_lookup_dict = {
dep: {
plat1: dep_info_a,
plat2: dep_info_b
},
dep1: {
plat1: dep_info_c
},
dep2: {
plat2: dep_info3
}
}
dep_manager._UpdateDependencies(base_config_mock)
self.assertEqual(expected_lookup_dict, dep_manager._lookup_dict)
self.assertFalse(dep_info1.Update.called)
self.assertFalse(dep_info2.Update.called)
self.assertFalse(dep_info3.Update.called)
self.assertFalse(dep_info_b.Update.called)
dep_info_a.Update.assert_called_once_with(dep_info1)
dep_info_c.Update.assert_called_once_with(dep_info2)
# Collision error.
dep_manager._lookup_dict = start_lookup_dict.copy()
dep_info = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep_info.dependency = dep
dep_info.platform = plat1
base_config_mock.IterDependencyInfo.return_value = iter([dep_info])
dep_info_a.Update.side_effect = ValueError
self.assertRaises(ValueError, dep_manager._UpdateDependencies,
base_config_mock)
# Ensure the testing data wasn't corrupted.
self.assertEqual(start_lookup_dict, {
dep: {
plat1: dep_info_a,
plat2: dep_info_b
},
dep1: {
plat1: dep_info_c
}
})
def testErrorInit(self):
with self.assertRaises(ValueError):
dependency_manager.DependencyManager(None)
with self.assertRaises(ValueError):
dependency_manager.DependencyManager('config_file?')
def testInitialUpdateDependencies(self):
dep_manager = dependency_manager.DependencyManager([])
# Empty BaseConfig.
dep_manager._lookup_dict = {}
base_config_mock = mock.MagicMock(spec=dependency_manager.BaseConfig)
base_config_mock.IterDependencyInfo.return_value = iter([])
dep_manager._UpdateDependencies(base_config_mock)
self.assertFalse(dep_manager._lookup_dict)
# One dependency/platform in a BaseConfig.
dep_manager._lookup_dict = {}
base_config_mock = mock.MagicMock(spec=dependency_manager.BaseConfig)
dep_info = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep = 'dependency'
plat = 'platform'
dep_info.dependency = dep
dep_info.platform = plat
base_config_mock.IterDependencyInfo.return_value = iter([dep_info])
expected_lookup_dict = {dep: {plat: dep_info}}
dep_manager._UpdateDependencies(base_config_mock)
self.assertEqual(expected_lookup_dict, dep_manager._lookup_dict)
self.assertFalse(dep_info.Update.called)
# One dependency multiple platforms in a BaseConfig.
dep_manager._lookup_dict = {}
base_config_mock = mock.MagicMock(spec=dependency_manager.BaseConfig)
dep = 'dependency'
plat1 = 'platform1'
plat2 = 'platform2'
dep_info1 = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep_info1.dependency = dep
dep_info1.platform = plat1
dep_info2 = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep_info2.dependency = dep
dep_info2.platform = plat2
base_config_mock.IterDependencyInfo.return_value = iter(
[dep_info1, dep_info2])
expected_lookup_dict = {dep: {plat1: dep_info1, plat2: dep_info2}}
dep_manager._UpdateDependencies(base_config_mock)
self.assertEqual(expected_lookup_dict, dep_manager._lookup_dict)
self.assertFalse(dep_info1.Update.called)
self.assertFalse(dep_info2.Update.called)
# Multiple dependencies, multiple platforms in a BaseConfig.
dep_manager._lookup_dict = {}
base_config_mock = mock.MagicMock(spec=dependency_manager.BaseConfig)
dep1 = 'dependency1'
dep2 = 'dependency2'
plat1 = 'platform1'
plat2 = 'platform2'
dep_info1 = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep_info1.dependency = dep1
dep_info1.platform = plat1
dep_info2 = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep_info2.dependency = dep1
dep_info2.platform = plat2
dep_info3 = mock.MagicMock(spec=dependency_manager.DependencyInfo)
dep_info3.dependency = dep2
dep_info3.platform = plat2
base_config_mock.IterDependencyInfo.return_value = iter(
[dep_info1, dep_info2, dep_info3])
expected_lookup_dict = {
dep1: {
plat1: dep_info1,
plat2: dep_info2
},
dep2: {
plat2: dep_info3
}
}
dep_manager._UpdateDependencies(base_config_mock)
self.assertEqual(expected_lookup_dict, dep_manager._lookup_dict)
self.assertFalse(dep_info1.Update.called)
self.assertFalse(dep_info2.Update.called)
self.assertFalse(dep_info3.Update.called)
def __init__(self, configs):
self._dependency_manager = dependency_manager.DependencyManager(
configs)
def testLocalPath(self):
dep_manager = dependency_manager.DependencyManager([])
# Empty lookup_dict
with self.assertRaises(exceptions.NoPathFoundError):
dep_manager.LocalPath('dep', 'plat')
def FetchBinaryDependencies(
platform, client_configs, fetch_reference_chrome_binary):
""" Fetch all binary dependenencies for the given |platform|.
Note: we don't fetch browser binaries by default because the size of the
binary is about 2Gb, and it requires cloud storage permission to
chrome-telemetry bucket.
Args:
platform: an instance of telemetry.core.platform
client_configs: A list of paths (string) to dependencies json files.
fetch_reference_chrome_binary: whether to fetch reference chrome binary for
the given platform.
"""
configs = [
dependency_manager.BaseConfig(TELEMETRY_PROJECT_CONFIG),
]
dep_manager = dependency_manager.DependencyManager(configs)
os_name = platform.GetOSName()
# If we're running directly on a Chrome OS device, fetch the binaries for
# linux instead, which should be compatible with CrOS. Otherwise, if we're
# running remotely on CrOS, fetch the binaries for the host platform like
# we do with android below.
if _IsChromeOSLocalMode(os_name):
os_name = 'linux'
target_platform = '%s_%s' % (os_name, platform.GetArchName())
dep_manager.PrefetchPaths(target_platform)
host_platform = None
fetch_devil_deps = False
if os_name in ('android', 'chromeos'):
host_platform = '%s_%s' % (
py_utils.GetHostOsName(), py_utils.GetHostArchName())
dep_manager.PrefetchPaths(host_platform)
if os_name == 'android':
if host_platform == 'linux_x86_64':
fetch_devil_deps = True
else:
logging.error('Devil only supports 64 bit linux as a host platform. '
'Android tests may fail.')
if fetch_reference_chrome_binary:
_FetchReferenceBrowserBinary(platform)
# For now, handle client config separately because the BUILD.gn & .isolate of
# telemetry tests in chromium src failed to include the files specified in its
# client config.
# (https://github.com/catapult-project/catapult/issues/2192)
# For now this is ok because the client configs usually don't include cloud
# storage infos.
# TODO(nednguyen): remove the logic of swallowing exception once the issue is
# fixed on Chromium side.
if client_configs:
manager = dependency_manager.DependencyManager(
list(dependency_manager.BaseConfig(c) for c in client_configs))
try:
manager.PrefetchPaths(target_platform)
if host_platform is not None:
manager.PrefetchPaths(host_platform)
except dependency_manager.NoPathFoundError as e:
logging.error('Error when trying to prefetch paths for %s: %s',
target_platform, e.message)
if fetch_devil_deps:
devil_env.config.Initialize()
devil_env.config.PrefetchPaths(arch=platform.GetArchName())
devil_env.config.PrefetchPaths()
def FetchBinaryDepdencies(platform, client_configs,
fetch_reference_chrome_binary):
""" Fetch all binary dependenencies for the given |platform|.
Note: we don't fetch browser binaries by default because the size of the
binary is about 2Gb, and it requires cloud storage permission to
chrome-telemetry bucket.
Args:
platform: an instance of telemetry.core.platform
client_configs: A list of paths (string) to dependencies json files.
fetch_reference_chrome_binary: whether to fetch reference chrome binary for
the given platform.
"""
configs = [
dependency_manager.BaseConfig(TELEMETRY_PROJECT_CONFIG),
dependency_manager.BaseConfig(BATTOR_BINARY_CONFIG)
]
dep_manager = dependency_manager.DependencyManager(configs)
target_platform = '%s_%s' % (platform.GetOSName(), platform.GetArchName())
dep_manager.PrefetchPaths(target_platform)
host_platform = None
fetch_devil_deps = False
if platform.GetOSName() == 'android':
host_platform = '%s_%s' % (platform_module.GetHostPlatform().GetOSName(
), platform_module.GetHostPlatform().GetArchName())
dep_manager.PrefetchPaths(host_platform)
# TODO(aiolos): this is a hack to prefetch the devil deps.
if host_platform == 'linux_x86_64':
fetch_devil_deps = True
else:
logging.error(
'Devil only supports 64 bit linux as a host platform. '
'Android tests may fail.')
if fetch_reference_chrome_binary:
_FetchReferenceBrowserBinary(platform)
# For now, handle client config separately because the BUILD.gn & .isolate of
# telemetry tests in chromium src failed to include the files specified in its
# client config.
# (https://github.com/catapult-project/catapult/issues/2192)
# For now this is ok because the client configs usually don't include cloud
# storage infos.
# TODO(nednguyen): remove the logic of swallowing exception once the issue is
# fixed on Chromium side.
if client_configs:
manager = dependency_manager.DependencyManager(
list(dependency_manager.BaseConfig(c) for c in client_configs))
try:
manager.PrefetchPaths(target_platform)
if host_platform is not None:
manager.PrefetchPaths(host_platform)
except dependency_manager.NoPathFoundError as e:
logging.error('Error when trying to prefetch paths for %s: %s',
target_platform, e.message)
# TODO(aiolos, jbudorick): we should have a devil pre-fetch API to call here
# and/or switch devil to use the same platform names so we can include it in
# the primary loop.
if fetch_devil_deps:
devil_env.config.Initialize()
devil_env.config._dm.PrefetchPaths(target_platform)
devil_env.config._dm.PrefetchPaths('linux2_x86_64')