def _RetrieveManifest(repo_url, revision, os_platform): # pragma: no cover. """Returns the manifest of all the dependencies for the given revision. Args: repo_url (str): The url to the Gitiles project of the root repository. revision (str): The revision of the root repository. os_platform (str): The platform of the code checkout. Returns: A list of DependencyRepository instances ordered reversely by the relative path of each dependency checkout in the checkout of the root repository. The longer the relative path, the smaller index in the returned list. The reverse order is to make it easy to reliably determine which dependency a file is from, when given a file path relative to the root repository. """ manifest = [] root_dir = 'src/' def AddDependencyToManifest(path, url, revision): # pragma: no cover. if path.startswith(root_dir): path = path[len(root_dir):] assert not path.startswith('//') path = '//' + path if not path.endswith('/'): path = path + '/' # Parse the url to extract the hostname and project name. # For "https://chromium.google.com/chromium/src.git", we get # ParseResult(netloc='chromium.google.com', path='/chromium/src.git', ...) result = urlparse.urlparse(url) assert result.path, 'No project extracted from %s' % url manifest.append( DependencyRepository( path=path, server_host=result.netloc, project=result.path[1:], # Strip the leading '/'. revision=revision)) # Add the root repository. AddDependencyToManifest('src/', repo_url, revision) # Add all the dependent repositories. # DEPS fetcher now assumes chromium/src and master branch. dep_fetcher = chrome_dependency_fetcher.ChromeDependencyFetcher( CachedGitilesRepository.Factory(FinditHttpClient())) deps = dep_fetcher.GetDependency(revision, os_platform) for path, dep in deps.iteritems(): # Remove clause when crbug.com/929315 gets fixed. if path in _BLACKLISTED_DEPS.get(repo_url, []): continue AddDependencyToManifest(path, dep.repo_url, dep.revision) manifest.sort(key=lambda x: len(x.path), reverse=True) return manifest
def _GetDependencies(chromium_revision, os_platform): """Returns the dependencies used by the specified chromium revision.""" deps = {} dep_fetcher = chrome_dependency_fetcher.ChromeDependencyFetcher( CachedGitilesRepository.Factory(HttpClientAppengine())) for path, dependency in dep_fetcher.GetDependency(chromium_revision, os_platform).iteritems(): deps[path] = { 'repo_url': dependency.repo_url, 'revision': dependency.revision, } return deps
def ExtractDepsInfo(failure_info, change_logs): """ Args: failure_info (BaseFailureInfo): Information about all build failures. change_logs (dict): Result of PullChangeLogs(). Returns: A dict with the following form: { 'deps': { 'path/to/dependency': { 'revision': 'git_hash', 'repo_url': 'https://url/to/dependency/repo.git', }, ... }, 'deps_rolls': { 'git_revision': [ { 'path': 'src/path/to/dependency', 'repo_url': 'https://url/to/dependency/repo.git', 'new_revision': 'git_hash1', 'old_revision': 'git_hash2', }, ... ], ... } } """ chromium_revision = failure_info.chromium_revision os_platform = GetOSPlatformName(failure_info.master_name, failure_info.builder_name) dep_fetcher = chrome_dependency_fetcher.ChromeDependencyFetcher( CachedGitilesRepository.Factory(FinditHttpClient())) return { 'deps': GetDependencies(chromium_revision, os_platform, dep_fetcher), 'deps_rolls': DetectDependencyRolls(change_logs, os_platform, dep_fetcher) }
def _DetectDependencyRolls(change_logs, os_platform): """Detect DEPS rolls in the given CL change logs. Args: change_logs (dict): Output of pipeline PullChangelogPipeline.run(). Returns: A dict in the following form: { 'git_revision': [ { 'path': 'src/path/to/dependency/', 'repo_url': 'https://url/to/dependency/repo.git', 'new_revision': 'git_hash1', 'old_revision': 'git_hash2', }, ... ], ... } """ deps_rolls = {} dep_fetcher = chrome_dependency_fetcher.ChromeDependencyFetcher( CachedGitilesRepository.Factory(HttpClientAppengine())) for revision, change_log in change_logs.iteritems(): # Check DEPS roll only if the chromium DEPS file is changed by the CL. for touched_file in change_log['touched_files']: if touched_file['new_path'] == 'DEPS': # In git, r^ refers to the previous revision of r. old_revision = '%s^' % revision rolls = dep_fetcher.GetDependencyRolls(old_revision, revision, os_platform) deps_rolls[revision] = [roll.ToDict() for roll in rolls] break return deps_rolls
class ChromiumDEPSTest(testing.AppengineTestCase): deps_downloader = chrome_dependency_fetcher.DEPSDownloader( gitiles_repository.GitilesRepository.Factory(HttpClientAppengine())) chrome_dep_fetcher = chrome_dependency_fetcher.ChromeDependencyFetcher( gitiles_repository.GitilesRepository.Factory(HttpClientAppengine())) @mock.patch.object(GitilesRepository, 'GetSource', side_effect=[DEPS_CONTENT]) def testUseDEPS(self, mock_source): revision = 'abc' content = self.deps_downloader.Load( chrome_dependency_fetcher._CHROMIUM_REPO_MASTER, revision, 'DEPS') self.assertEqual(DEPS_CONTENT, content) mock_source.assert_called_once_with(DEPS, revision) @mock.patch.object(GitilesRepository, 'GetSource', side_effect=[None, DEPS_GIT_CONTENT]) def testUseDEPS_GIT(self, mock_source): revision = 'abc' content = self.deps_downloader.Load( chrome_dependency_fetcher._CHROMIUM_REPO_MASTER, revision, 'DEPS') self.assertEqual(DEPS_GIT_CONTENT, content) mock_source.assert_has_calls( [mock.call(DEPS, revision), mock.call(DEPS_GIT, revision)]) @mock.patch.object(GitilesRepository, 'GetSource', side_effect=[DEPS_CONTENT]) def testNotChromiumRepo(self, mock_source): revision = 'abc' content = self.deps_downloader.Load('https://src.git', revision, 'DEPS') self.assertEqual(DEPS_CONTENT, content) mock_source.assert_called_once_with(DEPS, revision) @mock.patch.object(GitilesRepository, 'GetSource', side_effect=[None, DEPS_CONTENT]) def testUseDEPSForNonexistentDEPS(self, mock_source): revision = 'abc' content = self.deps_downloader.Load('https://src.git', revision, 'NONEXISTENT_DEPS') self.assertEqual(DEPS_CONTENT, content) mock_source.assert_has_calls([ mock.call('NONEXISTENT_DEPS', revision), mock.call(DEPS, revision) ]) @mock.patch.object(GitilesRepository, 'GetSource') def testUseSlaveDEPS(self, mock_source): revision = 'abc' expected_content = 'slave DEPS content' mock_source.side_effect = [expected_content] content = self.deps_downloader.Load('https://src.git', revision, 'slave.DEPS') self.assertEqual(expected_content, content) mock_source.assert_called_once_with('slave.DEPS', revision) @mock.patch.object(GitilesRepository, 'GetSource', return_value=None) def testFailedToPullDEPSFile(self, _): self.assertRaisesRegexp(Exception, 'Failed to pull DEPS file.', self.deps_downloader.Load, 'https://src.git', 'abc', 'DEPS') def testDEPSDownloaderForChromeVersion(self): def _MockGet(*_): return 200, base64.b64encode('Dummy DEPS content'), {} self.mock(HttpClientAppengine, '_Get', _MockGet) deps_downloader = chrome_dependency_fetcher.DEPSDownloader( gitiles_repository.GitilesRepository.Factory( HttpClientAppengine())) content = deps_downloader.Load('http://chrome-internal', '50.0.1234.0', 'DEPS') self.assertEqual(content, 'Dummy DEPS content') def testGetDependency(self): src_path = 'src' src_repo_url = 'https://chromium.googlesource.com/chromium/src.git' src_revision = '123a' os_platform = 'unix' child1_dep = Dependency('src/a', 'https://a.git', '123a', 'DEPS') child2_dep = Dependency('src/b', 'https://b.git', '123b', 'DEPS') grand_child1 = Dependency('src/a/aa', 'https://aa.git', '123aa', 'DEPS') expected_dependency_dict = { 'src/a': child1_dep, 'src/b': child2_dep, 'src/a/aa': grand_child1, } def DummyUpdateDependencyTree(root_dep, target_os_list, _): self.assertEqual(src_path, root_dep.path) self.assertEqual(src_repo_url, root_dep.repo_url) self.assertEqual(src_revision, root_dep.revision) self.assertEqual([os_platform], target_os_list) expected_dependency_dict[root_dep.path] = root_dep child1_dep.SetParent(root_dep) child2_dep.SetParent(root_dep) grand_child1.SetParent(child1_dep) self.mock(deps_parser, 'UpdateDependencyTree', DummyUpdateDependencyTree) dependency_dict = self.chrome_dep_fetcher.GetDependency( src_revision, os_platform) self.assertEqual(expected_dependency_dict, dependency_dict) def testGetDependencyForChromeVersion(self): src_path = 'src' src_repo_url = 'https://chromium.googlesource.com/chromium/src.git' os_platform = 'unix' child1_dep = Dependency('src/a', 'https://a.git', '123a', 'DEPS') child2_dep = Dependency('src/b', 'https://b.git', '123b', 'DEPS') grand_child1 = Dependency('src/a/aa', 'https://aa.git', '123aa', 'DEPS') expected_dependency_dict = { 'src/a': child1_dep, 'src/b': child2_dep, 'src/a/aa': grand_child1, } def DummyUpdateDependencyTree(root_dep, target_os_list, _): self.assertEqual(src_path, root_dep.path) self.assertEqual(src_repo_url, root_dep.repo_url) self.assertEqual([os_platform], target_os_list) expected_dependency_dict[root_dep.path] = root_dep child1_dep.SetParent(root_dep) child2_dep.SetParent(root_dep) grand_child1.SetParent(child1_dep) self.mock(deps_parser, 'UpdateDependencyTree', DummyUpdateDependencyTree) dependency_dict = self.chrome_dep_fetcher.GetDependency( '50.0.1234.0', os_platform) self.assertEqual(expected_dependency_dict, dependency_dict) def testGetDependencyRolls(self): def MockGetDependency(revision, os_platform, _=False): self.assertEqual('unix', os_platform) if revision == 'rev2': return { 'src': Dependency('src', 'https://url_src', 'rev2', 'DEPS'), 'src/dep1': Dependency('src/dep1', 'https://url_dep1', '9', 'DEPS'), 'src/dep2': Dependency('src/dep2', 'https://url_dep2', '5', 'DEPS'), 'src/dep4': Dependency('src/dep4', 'https://url_dep4', '1', 'DEPS'), } else: self.assertEqual('rev1', revision) return { 'src': Dependency('src', 'https://url_src', 'rev1', 'DEPS'), 'src/dep1': Dependency('src/dep1', 'https://url_dep1', '7', 'DEPS'), 'src/dep2': Dependency('src/dep2', 'https://url_dep2', '5', 'DEPS'), 'src/dep3': Dependency('src/dep3', 'https://url_dep3', '3', 'DEPS'), } self.mock(self.chrome_dep_fetcher, 'GetDependency', MockGetDependency) expected_deps_rolls = [ { 'path': 'src/dep1', 'repo_url': 'https://url_dep1', 'old_revision': '7', 'new_revision': '9', }, { 'path': 'src/dep4', 'repo_url': 'https://url_dep4', 'old_revision': None, 'new_revision': '1', }, { 'path': 'src/dep3', 'repo_url': 'https://url_dep3', 'old_revision': '3', 'new_revision': None, }, ] deps_rolls = self.chrome_dep_fetcher.GetDependencyRolls( 'rev1', 'rev2', 'unix') deps_rolls = [roll.ToDict() for roll in deps_rolls] deps_rolls.sort() expected_deps_rolls.sort() self.assertEqual(expected_deps_rolls, deps_rolls) def testGetDependencyRollsDict(self): def _MockGetDependencyRolls(old_revision, new_revision, *_, **kargs): dependency_rolls = [ DependencyRoll('src/dep1', 'https://url_dep1', '7', '9'), DependencyRoll('src/dep2', 'https://url_dep2', '3', None) ] if not kargs['skip_chromium_roll']: dependency_rolls.append( DependencyRoll( 'src', 'https://chromium.googlesource.com/chromium/src.git', old_revision, new_revision)) return dependency_rolls self.mock(self.chrome_dep_fetcher, 'GetDependencyRolls', _MockGetDependencyRolls) expected_deps_rolls_skip_chromium = [ DependencyRoll('src/dep1', 'https://url_dep1', '7', '9'), DependencyRoll('src/dep2', 'https://url_dep2', '3', None) ] self.assertEqual( self.chrome_dep_fetcher.GetDependencyRolls( '4', '5', 'all', skip_chromium_roll=True), expected_deps_rolls_skip_chromium) expected_deps_rolls_dict = { 'src/dep1': DependencyRoll('src/dep1', 'https://url_dep1', '7', '9'), 'src/dep2': DependencyRoll('src/dep2', 'https://url_dep2', '3', None), 'src': DependencyRoll( 'src', 'https://chromium.googlesource.com/chromium/src.git', '4', '5'), } self.assertEqual( self.chrome_dep_fetcher.GetDependencyRollsDict('4', '5', 'all'), expected_deps_rolls_dict)
# found in the LICENSE file. import logging import mock from testing_utils import testing from common.findit_http_client import FinditHttpClient from gae_libs.gitiles.cached_gitiles_repository import CachedGitilesRepository from libs.deps import chrome_dependency_fetcher from libs.deps.dependency import Dependency from libs.gitiles.diff import ChangeType from services import deps from services.parameters import TestFailureInfo from services.test.build_failure_analysis_test import ChangeLogFromDict _DEP_FETCHER = chrome_dependency_fetcher.ChromeDependencyFetcher( CachedGitilesRepository.Factory(FinditHttpClient())) class DepsTest(testing.AppengineTestCase): def testGetOSPlatformName(self): master_name = 'chromium.linux' builder_name = 'android' self.assertEqual('android', deps.GetOSPlatformName(master_name, builder_name)) def testGetOSPlatformNameDefault(self): master_name = 'chromium.linux' builder_name = 'linux' self.assertEqual('unix', deps.GetOSPlatformName(master_name, builder_name))
class ChromiumDEPSTest(testing.AppengineTestCase): DEPS_GIT = '.DEPS.git' DEPS = 'DEPS' deps_downloader = chrome_dependency_fetcher.DEPSDownloader( MockGitilesRepositoryFactory) chrome_dep_fetcher = chrome_dependency_fetcher.ChromeDependencyFetcher( MockGitilesRepositoryFactory) def testUseDEPS_GIT(self): revision = 'abc' expected_content = '.DEPS.git content' MockGitilesRepository.RESPONSES = { self.DEPS_GIT: { revision: expected_content }, self.DEPS: { revision: 'DEPS test' }, } content = self.deps_downloader.Load( chrome_dependency_fetcher._CHROMIUM_REPO_MASTER, revision, 'DEPS') self.assertEqual(expected_content, content) def testNotUseDEPS_GIT(self): revision = 'abc' expected_content = 'DEPS test' MockGitilesRepository.RESPONSES = { self.DEPS_GIT: { revision: '.DEPS.git content' }, self.DEPS: { revision: expected_content }, } self.mock(gitiles_repository, 'GitilesRepository', MockGitilesRepository) content = self.deps_downloader.Load('https://src.git', revision, 'DEPS') self.assertEqual(expected_content, content) def testUseFallbackDEPS(self): revision = 'abc' expected_content = 'DEPS test' MockGitilesRepository.RESPONSES = { self.DEPS: { revision: expected_content }, } self.mock(gitiles_repository, 'GitilesRepository', MockGitilesRepository) content = self.deps_downloader.Load('https://src.git', revision, 'NONEXISTENT_DEPS') self.assertEqual(expected_content, content) def testUseSlaveDEPS(self): revision = 'abc' expected_content = 'slave DEPS content' MockGitilesRepository.RESPONSES = { self.DEPS_GIT: { revision: '.DEPS.git content' }, 'slave.DEPS': { revision: expected_content }, } self.mock(gitiles_repository, 'GitilesRepository', MockGitilesRepository) content = self.deps_downloader.Load('https://src.git', revision, 'slave.DEPS') self.assertEqual(expected_content, content) def testFailedToPullDEPSFile(self): MockGitilesRepository.RESPONSES = {} self.assertRaisesRegexp(Exception, 'Failed to pull DEPS file.', self.deps_downloader.Load, 'https://src.git', 'abc', 'DEPS') def testDEPSDownloaderForChromeVersion(self): def _MockGet(*_): return 200, base64.b64encode('Dummy DEPS content') self.mock(HttpClientAppengine, '_Get', _MockGet) deps_downloader = chrome_dependency_fetcher.DEPSDownloader( gitiles_repository.GitilesRepository.Factory( HttpClientAppengine())) content = deps_downloader.Load('http://chrome-internal', '50.0.1234.0', 'DEPS') self.assertEqual(content, 'Dummy DEPS content') self.assertRaisesRegexp( Exception, 'Failed to pull DEPS file from http://chrome, at revision 50.0.1234.1.', self.deps_downloader.Load, 'http://chrome', '50.0.1234.1', 'DEPS') def testGetDependency(self): src_path = 'src/' src_repo_url = 'https://chromium.googlesource.com/chromium/src.git' src_revision = '123a' os_platform = 'unix' child1_dep = Dependency('src/a/', 'https://a.git', '123a', 'DEPS') child2_dep = Dependency('src/b/', 'https://b.git', '123b', 'DEPS') grand_child1 = Dependency('src/a/aa/', 'https://aa.git', '123aa', 'DEPS') expected_dependency_dict = { 'src/a/': child1_dep, 'src/b/': child2_dep, 'src/a/aa/': grand_child1, } def DummyUpdateDependencyTree(root_dep, target_os_list, _): self.assertEqual(src_path, root_dep.path) self.assertEqual(src_repo_url, root_dep.repo_url) self.assertEqual(src_revision, root_dep.revision) self.assertEqual([os_platform], target_os_list) expected_dependency_dict[root_dep.path] = root_dep child1_dep.SetParent(root_dep) child2_dep.SetParent(root_dep) grand_child1.SetParent(child1_dep) self.mock(deps_parser, 'UpdateDependencyTree', DummyUpdateDependencyTree) dependency_dict = self.chrome_dep_fetcher.GetDependency( src_revision, os_platform) self.assertEqual(expected_dependency_dict, dependency_dict) def testGetDependencyForChromeVersion(self): src_path = 'src/' src_repo_url = 'https://chromium.googlesource.com/chromium/src.git' os_platform = 'unix' child1_dep = Dependency('src/a/', 'https://a.git', '123a', 'DEPS') child2_dep = Dependency('src/b/', 'https://b.git', '123b', 'DEPS') grand_child1 = Dependency('src/a/aa/', 'https://aa.git', '123aa', 'DEPS') expected_dependency_dict = { 'src/a/': child1_dep, 'src/b/': child2_dep, 'src/a/aa/': grand_child1, } def DummyUpdateDependencyTree(root_dep, target_os_list, _): self.assertEqual(src_path, root_dep.path) self.assertEqual(src_repo_url, root_dep.repo_url) self.assertEqual([os_platform], target_os_list) expected_dependency_dict[root_dep.path] = root_dep child1_dep.SetParent(root_dep) child2_dep.SetParent(root_dep) grand_child1.SetParent(child1_dep) self.mock(gitiles_repository, 'GitilesRepository', MockGitilesRepository) self.mock(deps_parser, 'UpdateDependencyTree', DummyUpdateDependencyTree) dependency_dict = self.chrome_dep_fetcher.GetDependency( '50.0.1234.0', os_platform) self.assertEqual(expected_dependency_dict, dependency_dict) def testGetDependencyRolls(self): def MockGetDependency(revision, os_platform, _=False): self.assertEqual('unix', os_platform) if revision == 'rev2': return { 'src/': Dependency('src/', 'https://url_src', 'rev2', 'DEPS'), 'src/dep1': Dependency('src/dep1', 'https://url_dep1', '9', 'DEPS'), 'src/dep2': Dependency('src/dep2', 'https://url_dep2', '5', 'DEPS'), 'src/dep4': Dependency('src/dep4', 'https://url_dep4', '1', 'DEPS'), } else: self.assertEqual('rev1', revision) return { 'src/': Dependency('src/', 'https://url_src', 'rev1', 'DEPS'), 'src/dep1': Dependency('src/dep1', 'https://url_dep1', '7', 'DEPS'), 'src/dep2': Dependency('src/dep2', 'https://url_dep2', '5', 'DEPS'), 'src/dep3': Dependency('src/dep3', 'https://url_dep3', '3', 'DEPS'), } self.mock(self.chrome_dep_fetcher, 'GetDependency', MockGetDependency) expected_deps_rolls = [ { 'path': 'src/dep1', 'repo_url': 'https://url_dep1', 'old_revision': '7', 'new_revision': '9', }, { 'path': 'src/dep4', 'repo_url': 'https://url_dep4', 'old_revision': None, 'new_revision': '1', }, { 'path': 'src/dep3', 'repo_url': 'https://url_dep3', 'old_revision': '3', 'new_revision': None, }, ] deps_rolls = self.chrome_dep_fetcher.GetDependencyRolls( 'rev1', 'rev2', 'unix') self.assertEqual(expected_deps_rolls, [roll.ToDict() for roll in deps_rolls]) def testGetDependencyRollsDict(self): def _MockGetDependencyRolls(old_revision, new_revision, *_, **kargs): dependency_rolls = [ DependencyRoll('src/dep1', 'https://url_dep1', '7', '9'), DependencyRoll('src/dep2', 'https://url_dep2', '3', None) ] if not kargs['skip_chromium_roll']: dependency_rolls.append( DependencyRoll( 'src/', 'https://chromium.googlesource.com/chromium/src.git', old_revision, new_revision)) return dependency_rolls self.mock(self.chrome_dep_fetcher, 'GetDependencyRolls', _MockGetDependencyRolls) expected_deps_rolls_skip_chromium = [ DependencyRoll('src/dep1', 'https://url_dep1', '7', '9'), DependencyRoll('src/dep2', 'https://url_dep2', '3', None) ] self.assertEqual( self.chrome_dep_fetcher.GetDependencyRolls( '4', '5', 'all', skip_chromium_roll=True), expected_deps_rolls_skip_chromium) expected_deps_rolls_dict = { 'src/dep1': DependencyRoll('src/dep1', 'https://url_dep1', '7', '9'), 'src/dep2': DependencyRoll('src/dep2', 'https://url_dep2', '3', None), 'src/': DependencyRoll( 'src/', 'https://chromium.googlesource.com/chromium/src.git', '4', '5'), } self.assertEqual( self.chrome_dep_fetcher.GetDependencyRollsDict('4', '5', 'all'), expected_deps_rolls_dict) def testIsChromeVersion(self): self.assertTrue( chrome_dependency_fetcher.IsChromeVersion('50.0.1234.1')) self.assertFalse(chrome_dependency_fetcher.IsChromeVersion('a.b.c.e')) self.assertFalse( chrome_dependency_fetcher.IsChromeVersion('5.021.2.0.123'))