def _RetrieveManifest(repo_url, revision, os_platform): # pragma: no cover.
"""Returns the manifest of all the dependencies for the given revision.
Args:
repo_url (str): The url to the Gitiles project of the root repository.
revision (str): The revision of the root repository.
os_platform (str): The platform of the code checkout.
Returns:
A list of DependencyRepository instances ordered reversely by the relative
path of each dependency checkout in the checkout of the root repository.
The longer the relative path, the smaller index in the returned list.
The reverse order is to make it easy to reliably determine which dependency
a file is from, when given a file path relative to the root repository.
"""
manifest = []
root_dir = 'src/'
def AddDependencyToManifest(path, url, revision): # pragma: no cover.
if path.startswith(root_dir):
path = path[len(root_dir):]
assert not path.startswith('//')
path = '//' + path
if not path.endswith('/'):
path = path + '/'
# Parse the url to extract the hostname and project name.
# For "https://chromium.google.com/chromium/src.git", we get
# ParseResult(netloc='chromium.google.com', path='/chromium/src.git', ...)
result = urlparse.urlparse(url)
assert result.path, 'No project extracted from %s' % url
manifest.append(
DependencyRepository(
path=path,
server_host=result.netloc,
project=result.path[1:], # Strip the leading '/'.
revision=revision))
# Add the root repository.
AddDependencyToManifest('src/', repo_url, revision)
# Add all the dependent repositories.
# DEPS fetcher now assumes chromium/src and master branch.
dep_fetcher = chrome_dependency_fetcher.ChromeDependencyFetcher(
CachedGitilesRepository.Factory(FinditHttpClient()))
deps = dep_fetcher.GetDependency(revision, os_platform)
for path, dep in deps.iteritems():
# Remove clause when crbug.com/929315 gets fixed.
if path in _BLACKLISTED_DEPS.get(repo_url, []):
continue
AddDependencyToManifest(path, dep.repo_url, dep.revision)
manifest.sort(key=lambda x: len(x.path), reverse=True)
return manifest
def _GetDependencies(chromium_revision, os_platform):
"""Returns the dependencies used by the specified chromium revision."""
deps = {}
dep_fetcher = chrome_dependency_fetcher.ChromeDependencyFetcher(
CachedGitilesRepository.Factory(HttpClientAppengine()))
for path, dependency in dep_fetcher.GetDependency(chromium_revision,
os_platform).iteritems():
deps[path] = {
'repo_url': dependency.repo_url,
'revision': dependency.revision,
}
return deps
def ExtractDepsInfo(failure_info, change_logs):
"""
Args:
failure_info (BaseFailureInfo): Information about all build failures.
change_logs (dict): Result of PullChangeLogs().
Returns:
A dict with the following form:
{
'deps': {
'path/to/dependency': {
'revision': 'git_hash',
'repo_url': 'https://url/to/dependency/repo.git',
},
...
},
'deps_rolls': {
'git_revision': [
{
'path': 'src/path/to/dependency',
'repo_url': 'https://url/to/dependency/repo.git',
'new_revision': 'git_hash1',
'old_revision': 'git_hash2',
},
...
],
...
}
}
"""
chromium_revision = failure_info.chromium_revision
os_platform = GetOSPlatformName(failure_info.master_name,
failure_info.builder_name)
dep_fetcher = chrome_dependency_fetcher.ChromeDependencyFetcher(
CachedGitilesRepository.Factory(FinditHttpClient()))
return {
'deps': GetDependencies(chromium_revision, os_platform, dep_fetcher),
'deps_rolls': DetectDependencyRolls(change_logs, os_platform,
dep_fetcher)
}
def _DetectDependencyRolls(change_logs, os_platform):
"""Detect DEPS rolls in the given CL change logs.
Args:
change_logs (dict): Output of pipeline PullChangelogPipeline.run().
Returns:
A dict in the following form:
{
'git_revision': [
{
'path': 'src/path/to/dependency/',
'repo_url': 'https://url/to/dependency/repo.git',
'new_revision': 'git_hash1',
'old_revision': 'git_hash2',
},
...
],
...
}
"""
deps_rolls = {}
dep_fetcher = chrome_dependency_fetcher.ChromeDependencyFetcher(
CachedGitilesRepository.Factory(HttpClientAppengine()))
for revision, change_log in change_logs.iteritems():
# Check DEPS roll only if the chromium DEPS file is changed by the CL.
for touched_file in change_log['touched_files']:
if touched_file['new_path'] == 'DEPS':
# In git, r^ refers to the previous revision of r.
old_revision = '%s^' % revision
rolls = dep_fetcher.GetDependencyRolls(old_revision, revision,
os_platform)
deps_rolls[revision] = [roll.ToDict() for roll in rolls]
break
return deps_rolls
class ChromiumDEPSTest(testing.AppengineTestCase):
deps_downloader = chrome_dependency_fetcher.DEPSDownloader(
gitiles_repository.GitilesRepository.Factory(HttpClientAppengine()))
chrome_dep_fetcher = chrome_dependency_fetcher.ChromeDependencyFetcher(
gitiles_repository.GitilesRepository.Factory(HttpClientAppengine()))
@mock.patch.object(GitilesRepository,
'GetSource',
side_effect=[DEPS_CONTENT])
def testUseDEPS(self, mock_source):
revision = 'abc'
content = self.deps_downloader.Load(
chrome_dependency_fetcher._CHROMIUM_REPO_MASTER, revision, 'DEPS')
self.assertEqual(DEPS_CONTENT, content)
mock_source.assert_called_once_with(DEPS, revision)
@mock.patch.object(GitilesRepository,
'GetSource',
side_effect=[None, DEPS_GIT_CONTENT])
def testUseDEPS_GIT(self, mock_source):
revision = 'abc'
content = self.deps_downloader.Load(
chrome_dependency_fetcher._CHROMIUM_REPO_MASTER, revision, 'DEPS')
self.assertEqual(DEPS_GIT_CONTENT, content)
mock_source.assert_has_calls(
[mock.call(DEPS, revision),
mock.call(DEPS_GIT, revision)])
@mock.patch.object(GitilesRepository,
'GetSource',
side_effect=[DEPS_CONTENT])
def testNotChromiumRepo(self, mock_source):
revision = 'abc'
content = self.deps_downloader.Load('https://src.git', revision,
'DEPS')
self.assertEqual(DEPS_CONTENT, content)
mock_source.assert_called_once_with(DEPS, revision)
@mock.patch.object(GitilesRepository,
'GetSource',
side_effect=[None, DEPS_CONTENT])
def testUseDEPSForNonexistentDEPS(self, mock_source):
revision = 'abc'
content = self.deps_downloader.Load('https://src.git', revision,
'NONEXISTENT_DEPS')
self.assertEqual(DEPS_CONTENT, content)
mock_source.assert_has_calls([
mock.call('NONEXISTENT_DEPS', revision),
mock.call(DEPS, revision)
])
@mock.patch.object(GitilesRepository, 'GetSource')
def testUseSlaveDEPS(self, mock_source):
revision = 'abc'
expected_content = 'slave DEPS content'
mock_source.side_effect = [expected_content]
content = self.deps_downloader.Load('https://src.git', revision,
'slave.DEPS')
self.assertEqual(expected_content, content)
mock_source.assert_called_once_with('slave.DEPS', revision)
@mock.patch.object(GitilesRepository, 'GetSource', return_value=None)
def testFailedToPullDEPSFile(self, _):
self.assertRaisesRegexp(Exception, 'Failed to pull DEPS file.',
self.deps_downloader.Load, 'https://src.git',
'abc', 'DEPS')
def testDEPSDownloaderForChromeVersion(self):
def _MockGet(*_):
return 200, base64.b64encode('Dummy DEPS content'), {}
self.mock(HttpClientAppengine, '_Get', _MockGet)
deps_downloader = chrome_dependency_fetcher.DEPSDownloader(
gitiles_repository.GitilesRepository.Factory(
HttpClientAppengine()))
content = deps_downloader.Load('http://chrome-internal', '50.0.1234.0',
'DEPS')
self.assertEqual(content, 'Dummy DEPS content')
def testGetDependency(self):
src_path = 'src'
src_repo_url = 'https://chromium.googlesource.com/chromium/src.git'
src_revision = '123a'
os_platform = 'unix'
child1_dep = Dependency('src/a', 'https://a.git', '123a', 'DEPS')
child2_dep = Dependency('src/b', 'https://b.git', '123b', 'DEPS')
grand_child1 = Dependency('src/a/aa', 'https://aa.git', '123aa',
'DEPS')
expected_dependency_dict = {
'src/a': child1_dep,
'src/b': child2_dep,
'src/a/aa': grand_child1,
}
def DummyUpdateDependencyTree(root_dep, target_os_list, _):
self.assertEqual(src_path, root_dep.path)
self.assertEqual(src_repo_url, root_dep.repo_url)
self.assertEqual(src_revision, root_dep.revision)
self.assertEqual([os_platform], target_os_list)
expected_dependency_dict[root_dep.path] = root_dep
child1_dep.SetParent(root_dep)
child2_dep.SetParent(root_dep)
grand_child1.SetParent(child1_dep)
self.mock(deps_parser, 'UpdateDependencyTree',
DummyUpdateDependencyTree)
dependency_dict = self.chrome_dep_fetcher.GetDependency(
src_revision, os_platform)
self.assertEqual(expected_dependency_dict, dependency_dict)
def testGetDependencyForChromeVersion(self):
src_path = 'src'
src_repo_url = 'https://chromium.googlesource.com/chromium/src.git'
os_platform = 'unix'
child1_dep = Dependency('src/a', 'https://a.git', '123a', 'DEPS')
child2_dep = Dependency('src/b', 'https://b.git', '123b', 'DEPS')
grand_child1 = Dependency('src/a/aa', 'https://aa.git', '123aa',
'DEPS')
expected_dependency_dict = {
'src/a': child1_dep,
'src/b': child2_dep,
'src/a/aa': grand_child1,
}
def DummyUpdateDependencyTree(root_dep, target_os_list, _):
self.assertEqual(src_path, root_dep.path)
self.assertEqual(src_repo_url, root_dep.repo_url)
self.assertEqual([os_platform], target_os_list)
expected_dependency_dict[root_dep.path] = root_dep
child1_dep.SetParent(root_dep)
child2_dep.SetParent(root_dep)
grand_child1.SetParent(child1_dep)
self.mock(deps_parser, 'UpdateDependencyTree',
DummyUpdateDependencyTree)
dependency_dict = self.chrome_dep_fetcher.GetDependency(
'50.0.1234.0', os_platform)
self.assertEqual(expected_dependency_dict, dependency_dict)
def testGetDependencyRolls(self):
def MockGetDependency(revision, os_platform, _=False):
self.assertEqual('unix', os_platform)
if revision == 'rev2':
return {
'src':
Dependency('src', 'https://url_src', 'rev2', 'DEPS'),
'src/dep1':
Dependency('src/dep1', 'https://url_dep1', '9', 'DEPS'),
'src/dep2':
Dependency('src/dep2', 'https://url_dep2', '5', 'DEPS'),
'src/dep4':
Dependency('src/dep4', 'https://url_dep4', '1', 'DEPS'),
}
else:
self.assertEqual('rev1', revision)
return {
'src':
Dependency('src', 'https://url_src', 'rev1', 'DEPS'),
'src/dep1':
Dependency('src/dep1', 'https://url_dep1', '7', 'DEPS'),
'src/dep2':
Dependency('src/dep2', 'https://url_dep2', '5', 'DEPS'),
'src/dep3':
Dependency('src/dep3', 'https://url_dep3', '3', 'DEPS'),
}
self.mock(self.chrome_dep_fetcher, 'GetDependency', MockGetDependency)
expected_deps_rolls = [
{
'path': 'src/dep1',
'repo_url': 'https://url_dep1',
'old_revision': '7',
'new_revision': '9',
},
{
'path': 'src/dep4',
'repo_url': 'https://url_dep4',
'old_revision': None,
'new_revision': '1',
},
{
'path': 'src/dep3',
'repo_url': 'https://url_dep3',
'old_revision': '3',
'new_revision': None,
},
]
deps_rolls = self.chrome_dep_fetcher.GetDependencyRolls(
'rev1', 'rev2', 'unix')
deps_rolls = [roll.ToDict() for roll in deps_rolls]
deps_rolls.sort()
expected_deps_rolls.sort()
self.assertEqual(expected_deps_rolls, deps_rolls)
def testGetDependencyRollsDict(self):
def _MockGetDependencyRolls(old_revision, new_revision, *_, **kargs):
dependency_rolls = [
DependencyRoll('src/dep1', 'https://url_dep1', '7', '9'),
DependencyRoll('src/dep2', 'https://url_dep2', '3', None)
]
if not kargs['skip_chromium_roll']:
dependency_rolls.append(
DependencyRoll(
'src',
'https://chromium.googlesource.com/chromium/src.git',
old_revision, new_revision))
return dependency_rolls
self.mock(self.chrome_dep_fetcher, 'GetDependencyRolls',
_MockGetDependencyRolls)
expected_deps_rolls_skip_chromium = [
DependencyRoll('src/dep1', 'https://url_dep1', '7', '9'),
DependencyRoll('src/dep2', 'https://url_dep2', '3', None)
]
self.assertEqual(
self.chrome_dep_fetcher.GetDependencyRolls(
'4', '5', 'all', skip_chromium_roll=True),
expected_deps_rolls_skip_chromium)
expected_deps_rolls_dict = {
'src/dep1':
DependencyRoll('src/dep1', 'https://url_dep1', '7', '9'),
'src/dep2':
DependencyRoll('src/dep2', 'https://url_dep2', '3', None),
'src':
DependencyRoll(
'src', 'https://chromium.googlesource.com/chromium/src.git',
'4', '5'),
}
self.assertEqual(
self.chrome_dep_fetcher.GetDependencyRollsDict('4', '5', 'all'),
expected_deps_rolls_dict)
# found in the LICENSE file.
import logging
import mock
from testing_utils import testing
from common.findit_http_client import FinditHttpClient
from gae_libs.gitiles.cached_gitiles_repository import CachedGitilesRepository
from libs.deps import chrome_dependency_fetcher
from libs.deps.dependency import Dependency
from libs.gitiles.diff import ChangeType
from services import deps
from services.parameters import TestFailureInfo
from services.test.build_failure_analysis_test import ChangeLogFromDict
_DEP_FETCHER = chrome_dependency_fetcher.ChromeDependencyFetcher(
CachedGitilesRepository.Factory(FinditHttpClient()))
class DepsTest(testing.AppengineTestCase):
def testGetOSPlatformName(self):
master_name = 'chromium.linux'
builder_name = 'android'
self.assertEqual('android',
deps.GetOSPlatformName(master_name, builder_name))
def testGetOSPlatformNameDefault(self):
master_name = 'chromium.linux'
builder_name = 'linux'
self.assertEqual('unix',
deps.GetOSPlatformName(master_name, builder_name))
class ChromiumDEPSTest(testing.AppengineTestCase):
DEPS_GIT = '.DEPS.git'
DEPS = 'DEPS'
deps_downloader = chrome_dependency_fetcher.DEPSDownloader(
MockGitilesRepositoryFactory)
chrome_dep_fetcher = chrome_dependency_fetcher.ChromeDependencyFetcher(
MockGitilesRepositoryFactory)
def testUseDEPS_GIT(self):
revision = 'abc'
expected_content = '.DEPS.git content'
MockGitilesRepository.RESPONSES = {
self.DEPS_GIT: {
revision: expected_content
},
self.DEPS: {
revision: 'DEPS test'
},
}
content = self.deps_downloader.Load(
chrome_dependency_fetcher._CHROMIUM_REPO_MASTER, revision, 'DEPS')
self.assertEqual(expected_content, content)
def testNotUseDEPS_GIT(self):
revision = 'abc'
expected_content = 'DEPS test'
MockGitilesRepository.RESPONSES = {
self.DEPS_GIT: {
revision: '.DEPS.git content'
},
self.DEPS: {
revision: expected_content
},
}
self.mock(gitiles_repository, 'GitilesRepository',
MockGitilesRepository)
content = self.deps_downloader.Load('https://src.git', revision,
'DEPS')
self.assertEqual(expected_content, content)
def testUseFallbackDEPS(self):
revision = 'abc'
expected_content = 'DEPS test'
MockGitilesRepository.RESPONSES = {
self.DEPS: {
revision: expected_content
},
}
self.mock(gitiles_repository, 'GitilesRepository',
MockGitilesRepository)
content = self.deps_downloader.Load('https://src.git', revision,
'NONEXISTENT_DEPS')
self.assertEqual(expected_content, content)
def testUseSlaveDEPS(self):
revision = 'abc'
expected_content = 'slave DEPS content'
MockGitilesRepository.RESPONSES = {
self.DEPS_GIT: {
revision: '.DEPS.git content'
},
'slave.DEPS': {
revision: expected_content
},
}
self.mock(gitiles_repository, 'GitilesRepository',
MockGitilesRepository)
content = self.deps_downloader.Load('https://src.git', revision,
'slave.DEPS')
self.assertEqual(expected_content, content)
def testFailedToPullDEPSFile(self):
MockGitilesRepository.RESPONSES = {}
self.assertRaisesRegexp(Exception, 'Failed to pull DEPS file.',
self.deps_downloader.Load, 'https://src.git',
'abc', 'DEPS')
def testDEPSDownloaderForChromeVersion(self):
def _MockGet(*_):
return 200, base64.b64encode('Dummy DEPS content')
self.mock(HttpClientAppengine, '_Get', _MockGet)
deps_downloader = chrome_dependency_fetcher.DEPSDownloader(
gitiles_repository.GitilesRepository.Factory(
HttpClientAppengine()))
content = deps_downloader.Load('http://chrome-internal', '50.0.1234.0',
'DEPS')
self.assertEqual(content, 'Dummy DEPS content')
self.assertRaisesRegexp(
Exception,
'Failed to pull DEPS file from http://chrome, at revision 50.0.1234.1.',
self.deps_downloader.Load, 'http://chrome', '50.0.1234.1', 'DEPS')
def testGetDependency(self):
src_path = 'src/'
src_repo_url = 'https://chromium.googlesource.com/chromium/src.git'
src_revision = '123a'
os_platform = 'unix'
child1_dep = Dependency('src/a/', 'https://a.git', '123a', 'DEPS')
child2_dep = Dependency('src/b/', 'https://b.git', '123b', 'DEPS')
grand_child1 = Dependency('src/a/aa/', 'https://aa.git', '123aa',
'DEPS')
expected_dependency_dict = {
'src/a/': child1_dep,
'src/b/': child2_dep,
'src/a/aa/': grand_child1,
}
def DummyUpdateDependencyTree(root_dep, target_os_list, _):
self.assertEqual(src_path, root_dep.path)
self.assertEqual(src_repo_url, root_dep.repo_url)
self.assertEqual(src_revision, root_dep.revision)
self.assertEqual([os_platform], target_os_list)
expected_dependency_dict[root_dep.path] = root_dep
child1_dep.SetParent(root_dep)
child2_dep.SetParent(root_dep)
grand_child1.SetParent(child1_dep)
self.mock(deps_parser, 'UpdateDependencyTree',
DummyUpdateDependencyTree)
dependency_dict = self.chrome_dep_fetcher.GetDependency(
src_revision, os_platform)
self.assertEqual(expected_dependency_dict, dependency_dict)
def testGetDependencyForChromeVersion(self):
src_path = 'src/'
src_repo_url = 'https://chromium.googlesource.com/chromium/src.git'
os_platform = 'unix'
child1_dep = Dependency('src/a/', 'https://a.git', '123a', 'DEPS')
child2_dep = Dependency('src/b/', 'https://b.git', '123b', 'DEPS')
grand_child1 = Dependency('src/a/aa/', 'https://aa.git', '123aa',
'DEPS')
expected_dependency_dict = {
'src/a/': child1_dep,
'src/b/': child2_dep,
'src/a/aa/': grand_child1,
}
def DummyUpdateDependencyTree(root_dep, target_os_list, _):
self.assertEqual(src_path, root_dep.path)
self.assertEqual(src_repo_url, root_dep.repo_url)
self.assertEqual([os_platform], target_os_list)
expected_dependency_dict[root_dep.path] = root_dep
child1_dep.SetParent(root_dep)
child2_dep.SetParent(root_dep)
grand_child1.SetParent(child1_dep)
self.mock(gitiles_repository, 'GitilesRepository',
MockGitilesRepository)
self.mock(deps_parser, 'UpdateDependencyTree',
DummyUpdateDependencyTree)
dependency_dict = self.chrome_dep_fetcher.GetDependency(
'50.0.1234.0', os_platform)
self.assertEqual(expected_dependency_dict, dependency_dict)
def testGetDependencyRolls(self):
def MockGetDependency(revision, os_platform, _=False):
self.assertEqual('unix', os_platform)
if revision == 'rev2':
return {
'src/':
Dependency('src/', 'https://url_src', 'rev2', 'DEPS'),
'src/dep1':
Dependency('src/dep1', 'https://url_dep1', '9', 'DEPS'),
'src/dep2':
Dependency('src/dep2', 'https://url_dep2', '5', 'DEPS'),
'src/dep4':
Dependency('src/dep4', 'https://url_dep4', '1', 'DEPS'),
}
else:
self.assertEqual('rev1', revision)
return {
'src/':
Dependency('src/', 'https://url_src', 'rev1', 'DEPS'),
'src/dep1':
Dependency('src/dep1', 'https://url_dep1', '7', 'DEPS'),
'src/dep2':
Dependency('src/dep2', 'https://url_dep2', '5', 'DEPS'),
'src/dep3':
Dependency('src/dep3', 'https://url_dep3', '3', 'DEPS'),
}
self.mock(self.chrome_dep_fetcher, 'GetDependency', MockGetDependency)
expected_deps_rolls = [
{
'path': 'src/dep1',
'repo_url': 'https://url_dep1',
'old_revision': '7',
'new_revision': '9',
},
{
'path': 'src/dep4',
'repo_url': 'https://url_dep4',
'old_revision': None,
'new_revision': '1',
},
{
'path': 'src/dep3',
'repo_url': 'https://url_dep3',
'old_revision': '3',
'new_revision': None,
},
]
deps_rolls = self.chrome_dep_fetcher.GetDependencyRolls(
'rev1', 'rev2', 'unix')
self.assertEqual(expected_deps_rolls,
[roll.ToDict() for roll in deps_rolls])
def testGetDependencyRollsDict(self):
def _MockGetDependencyRolls(old_revision, new_revision, *_, **kargs):
dependency_rolls = [
DependencyRoll('src/dep1', 'https://url_dep1', '7', '9'),
DependencyRoll('src/dep2', 'https://url_dep2', '3', None)
]
if not kargs['skip_chromium_roll']:
dependency_rolls.append(
DependencyRoll(
'src/',
'https://chromium.googlesource.com/chromium/src.git',
old_revision, new_revision))
return dependency_rolls
self.mock(self.chrome_dep_fetcher, 'GetDependencyRolls',
_MockGetDependencyRolls)
expected_deps_rolls_skip_chromium = [
DependencyRoll('src/dep1', 'https://url_dep1', '7', '9'),
DependencyRoll('src/dep2', 'https://url_dep2', '3', None)
]
self.assertEqual(
self.chrome_dep_fetcher.GetDependencyRolls(
'4', '5', 'all', skip_chromium_roll=True),
expected_deps_rolls_skip_chromium)
expected_deps_rolls_dict = {
'src/dep1':
DependencyRoll('src/dep1', 'https://url_dep1', '7', '9'),
'src/dep2':
DependencyRoll('src/dep2', 'https://url_dep2', '3', None),
'src/':
DependencyRoll(
'src/', 'https://chromium.googlesource.com/chromium/src.git',
'4', '5'),
}
self.assertEqual(
self.chrome_dep_fetcher.GetDependencyRollsDict('4', '5', 'all'),
expected_deps_rolls_dict)
def testIsChromeVersion(self):
self.assertTrue(
chrome_dependency_fetcher.IsChromeVersion('50.0.1234.1'))
self.assertFalse(chrome_dependency_fetcher.IsChromeVersion('a.b.c.e'))
self.assertFalse(
chrome_dependency_fetcher.IsChromeVersion('5.021.2.0.123'))