def testAbbreviatedCommitHash(self, mock_get):
mock_response = mock.Mock()
mock_response.status_code = 200
mock_response.text = textwrap.dedent("""\
)]}'
{
"commit": "deadbabefacebeeffacefeedbeefabcdeffedcba",
"other": "irrelevant stuff"
}
""")
mock_get.side_effect = [mock_response]
deps_file_content = textwrap.dedent("""\
deps = {
"fount/a": "https://example.com/xyz/a@deadbabe",
}
""")
class FakeResolver(object):
def Resolve(self, _url, _ref):
return None
cut = deps2submodules.Deps2Submodules(deps_file_content,
FakeResolver(), 'fount/')
cut.Evaluate()
self.assertEqual(mock_get.call_count, 1)
return _pretty_print(cut._gitmodules)
def testAbbreviatedCommitHash_missingField(self, mock_get):
mock_response = mock.Mock()
mock_response.status_code = 200
mock_response.text = textwrap.dedent("""\
)]}'
{
"kommit": "deadbabefacebeeffacefeedbeefabcdeffedcba",
"other": "irrelevant stuff"
}
""")
mock_get.side_effect = [mock_response]
deps_file_content = textwrap.dedent("""\
deps = {
"fount/a": "https://example.com/xyz/a@deadbabe",
}
""")
class FakeResolver(object):
def Resolve(self, _url, _ref):
return None
cut = deps2submodules.Deps2Submodules(deps_file_content,
FakeResolver(), 'fount/')
with self.assertRaises(Exception):
cut.Evaluate()
def testExcludeByPathPrefix(self):
# https://stackoverflow.com/a/1412728/98761 discusses textwrap.dedent()
#
deps_file_content = textwrap.dedent("""\
deps = {
"abc/def": "https://chromium.googlesource.com/xyz/foo@5d0be3947e7e238f38516eddbe286a61d3ec4bc9",
"fount/foo": "https://chromium.googlesource.com/xyz/bar@10803d79ece6d8752be370163d342030a816cb8f",
}
""")
cut = deps2submodules.Deps2Submodules(deps_file_content, None,
'fount/')
cut.Evaluate()
return _pretty_print(cut._gitmodules)
def testPruneConflict(self):
deps_file_content = textwrap.dedent("""\
deps = {
"fount/quux": "https://chromium.googlesource.com/xyz/abc@db1a5a09f7ddd42f3ce395a761725516593fc4ff",
"fount/quux/conflict": "https://chromium.googlesource.com/xyz/def@00e6690f83a9260717b1d38ce9b9332d6986516d",
"fount/foo/bar": "https://chromium.googlesource.com/xyz/pqr@41d25f51c301c5eee3737998b0d86573e4e91b90",
"fount/foo/bar/baz": "https://chromium.googlesource.com/xyz/ghi@dc030e592c36bfffe129fe0d3af4fb30dde35704",
}
""")
cut = deps2submodules.Deps2Submodules(deps_file_content, None,
'fount/')
cut.Evaluate()
return _pretty_print(cut._gitmodules)
def testMissingSymbolicRef(self):
deps_file_content = textwrap.dedent("""\
deps = {
"fount/one": "https://chromium.googlesource.com/xyz/abc@refs/heads/rogue",
}
""")
class FakeResolver(object):
def Resolve(self, _url, _ref):
return None
cut = deps2submodules.Deps2Submodules(deps_file_content,
FakeResolver(), 'fount/')
with self.assertRaises(Exception):
cut.Evaluate()
def testUpdateSubmodules(self):
deps_file_content = textwrap.dedent("""\
deps = {
"fount/a": "https://example.com/xyz/a@deadbeefdeadbeefdeadbeefdeadbeefdeadbeef",
"fount/b": "https://example.com/xyz/b@cafebabecafebabecafebabecafebabecafebabe",
}
""")
cut = deps2submodules.Deps2Submodules(deps_file_content, None,
'fount/')
cut.Evaluate()
repo = TestRepo('repo', TestClock())
hsh = cut.UpdateSubmodules(repo, EMPTY_TREE)
tree_dump = repo.run('ls-tree', '-r', hsh)
file_dump = repo.run('cat-file', '-p', '%s:.gitmodules' % hsh)
return (file_dump, tree_dump)
def testElidedDeps(self):
deps_file_content = textwrap.dedent("""\
deps = {
"fount/quux": None,
"fount/foo/bar": {
'url': "https://chromium.googlesource.com/xyz/abc@41d25f51c301c5eee3737998b0d86573e4e91b90",
},
"fount/foo/baz": {
'url': "https://chromium.googlesource.com/xyz/pqr@1c9064284a24b3486015eafdb391b141c27ada2b",
'condition': 'checkout_google_internal'
},
}
""")
cut = deps2submodules.Deps2Submodules(deps_file_content, None,
'fount/')
cut.Evaluate()
return _pretty_print(cut._gitmodules)
def testAbbreviatedCommitHash_badStatus(self, mock_get):
mock_response = mock.Mock()
mock_response.status_code = 500
mock_response.text = 'Something went wrong.'
mock_get.side_effect = [mock_response]
deps_file_content = textwrap.dedent("""\
deps = {
"fount/a": "https://example.com/xyz/a@deadbabe",
}
""")
class FakeResolver(object):
def Resolve(self, _url, _ref):
return None
cut = deps2submodules.Deps2Submodules(deps_file_content,
FakeResolver(), 'fount/')
with self.assertRaises(Exception):
cut.Evaluate()
def testExtraSubmodules(self):
deps_file_content = textwrap.dedent("""\
deps = {
"abc/def": "https://chromium.googlesource.com/xyz/foo@5d0be3947e7e238f38516eddbe286a61d3ec4bc9",
"fount/foo": "https://chromium.googlesource.com/xyz/bar@10803d79ece6d8752be370163d342030a816cb8f",
}
""")
class FakeResolver(object):
def Resolve(self, url, ref):
assert (url, ref) == (
'https://chromium.googlesource.com/chromium/src/out',
'refs/heads/master')
return 'e5e08ac07d8a3530de282fbf99472c4c7625f949'
cut = deps2submodules.Deps2Submodules(
deps_file_content, FakeResolver(), 'fount/',
['fount/out=https://chromium.googlesource.com/chromium/src/out'])
cut.Evaluate()
return _pretty_print(cut._gitmodules)
def testSymbolicRef(self):
deps_file_content = textwrap.dedent("""\
deps = {
"fount/one": "https://chromium.googlesource.com/xyz/abc@refs/heads/rogue",
"fount/another": "https://chromium.googlesource.com/xyz/def",
}
""")
class FakeResolver(object):
def Resolve(self, url, ref):
if (url, ref) == ('https://chromium.googlesource.com/xyz/def',
'master'):
return '2a2337d1d2e5bfffa670a91f2231d5c428dfaadd'
elif (url,
ref) == ('https://chromium.googlesource.com/xyz/pqr',
'master'):
return 'a0cde59caf0889feb56aac1abd6eb5732ff0ddb8'
else:
assert (url, ref) == (
'https://chromium.googlesource.com/xyz/abc',
'refs/heads/rogue')
return 'cc103db39df46ec8fdce463279180d948ff6e81e'
cut = deps2submodules.Deps2Submodules(deps_file_content,
FakeResolver(), 'fount/')
cut.Evaluate()
first_result = _pretty_print(cut._gitmodules)
new_deps_file_content = textwrap.dedent("""\
deps = {
"fount/one": "https://chromium.googlesource.com/xyz/abc@refs/heads/rogue",
"fount/another": "https://chromium.googlesource.com/xyz/def",
"fount/yet/another": "https://chromium.googlesource.com/xyz/pqr@origin/master",
}
""")
cut = cut.withUpdatedDeps(new_deps_file_content)
cut.Evaluate()
second_result = _pretty_print(cut._gitmodules)
return (first_result, second_result)
def reify_submodules(origin_repo,
target,
dry_run=False,
limit=None,
extra_submodules=None,
epoch=None):
origin_repo.fetch()
shadow = repo.Repo(target)
shadow.dry_run = dry_run
shadow.repos_dir = origin_repo.repos_dir
shadow.reify(share_from=origin_repo)
synth_parent = shadow["refs/heads/master"].commit
if synth_parent is INVALID:
# If the shadow repo doesn't have any commits yet (we're just
# starting up for the first time), start at the beginning of history
# at the origin repo, either the absolute beginning, or the "epoch"
# at which we are configured to start.
if epoch:
processed = origin_repo.get_commit(epoch)
if processed is INVALID:
LOGGER.error("Requested epoch commit %s does not exist", epoch)
return False
else:
# For now, preserve the former behavior when this new "epoch" optional
# argument is missing. But once this new capability is deployed, the
# default behavior here should simply be to always start at the absolute
# beginning: i.e., there's no reason to default to the _CHROMIUM_SRC_EPOCH
# once that value can be explicitly passed in to us from the config.
#
# Note that looking up _CHROMIUM_SRC_EPOCH on repos other than
# chromium/src produces INVALID, and that's fine (it means we'll
# start at the beginning of history).
processed = origin_repo.get_commit(_CHROMIUM_SRC_EPOCH)
else:
footers = synth_parent.data.footers
if MIRRORED_COMMIT not in footers:
LOGGER.error('No footers for synthesized commit %r',
synth_parent.hsh)
return False
processed_commit_hash = footers[MIRRORED_COMMIT][0]
processed = origin_repo.get_commit(processed_commit_hash)
if processed is INVALID:
LOGGER.error(
'Mirrored commit %s (from synth. commit %r) does not exist',
processed_commit_hash, synth_parent.hsh)
return False
LOGGER.info('starting with tree %r', synth_parent.data.tree)
count = 0
known_hash = ''
submods = None
path_prefix = '%s/' % _Humanish(origin_repo.url)
resolver = deps2submodules.GitRefResolver(shadow)
commits = origin_repo[processed.hsh].to(origin_repo[REF_NAME],
'',
first_parent=True)
for commit in itertools.islice(commits, limit):
LOGGER.info("at commit %s" % commit.hsh)
original_tree = commit.data.to_dict()['tree']
deps_info = origin_repo.run('ls-tree', commit.hsh, '--',
'DEPS').split()
if len(deps_info):
# ls-tree output looks something like this:
# 100644 blob 726d435df65288b740ea55d38a5070c9870b8172 DEPS
deps_hash = deps_info[2]
assert SHA1_RE.match(deps_hash)
if deps_hash == known_hash:
# DEPS file has not changed in this commit: no need to repeat
# the parsing/processing of it.
#
# TODO: When we add support for recurse-deps, either remove or refine
# this optimization. (Even if top-level DEPS hasn't changed, the DEPS
# of a submodule might have.)
pass
else:
# new version of DEPS file
deps_object_path = '%s:DEPS' % commit.hsh
deps_content = origin_repo.run('cat-file', 'blob',
deps_object_path)
if known_hash:
LOGGER.debug('commit %s has modified DEPS file' %
commit.hsh)
submods = submods.withUpdatedDeps(deps_content)
else:
submods = deps2submodules.Deps2Submodules(
deps_content, resolver, path_prefix, extra_submodules)
submods.Evaluate()
known_hash = deps_hash
try:
new_tree = submods.UpdateSubmodules(shadow, commit.hsh)
except Exception as e: # pragma: no cover
# TODO: build a wrapping exception the proper way
LOGGER.error("exception happened on %s: %s" %
(commit.hsh, str(e)))
raise
else:
# DEPS file does not even exist. This should only happen in the
# very earliest commits of repos (and never in chromium/src,
# because we went to the trouble of finding epoch).
LOGGER.warn('commit %s has no DEPS file' % commit.hsh)
new_tree = original_tree
footers = [(MIRRORED_COMMIT, [commit.hsh])]
data = commit.data.alter(
parents=[synth_parent.hsh] if synth_parent is not INVALID else [],
tree=new_tree,
footers=collections.OrderedDict(footers))
synth_parent = shadow.get_commit(shadow.intern(data, 'commit'))
count += 1
if count:
output = shadow.fast_forward_push({shadow[REF_NAME]: synth_parent},
include_err=True,
timeout=PUSH_TIMEOUT)
if output: # pragma: no cover
LOGGER.info(output)
return True
