def content_of(commit):
"""Calculates the content of ``commit`` such that a gnumbd-landed commit and
the original commit will compare as equals. Returns the content as a
git2.CommitData object.
This strips out:
* The parent(s)
* The committer date
* footers beginning with 'Cr-'
* the 'git-svn-id' footer.
* the '(cherry picked from ...)' line.
Stores a cached copy of the result data on the ``commit`` instance itself.
"""
if commit is None:
return git2.INVALID
if not hasattr(commit, '_cr_content'):
d = commit.data
d = tweak_cherry_pick(d)
footers = infra_types.thaw(d.footers)
footers[GIT_SVN_ID] = None
for k in footers.keys():
if k.startswith(FOOTER_PREFIX):
footers[k] = None
commit._cr_content = d.alter(
parents=(),
committer=d.committer.alter(timestamp=git2.data.NULL_TIMESTAMP),
footers=footers)
return commit._cr_content # pylint: disable=W0212
def fix_footers(origin, run, checkpoint, mirrors, **_):
branch = origin['refs/heads/branch']
branch.make_commit(
'sweet commit',
{'mirrored_path': {'sweet': 'totally!'}},
OrderedDict([
('git-svn-id', ['totally annoying!']),
('Cr-Commit-Position', ['refs/heads/branch@{#12345}']),
('Commit-Id', ['whaaat']),
('Cr-Branched-From', ['deadbeef-refs/heads/master@{#12300}']),
]))
checkpoint('a really sweet commit')
run()
checkpoint('a really sweet (mirrored) commit')
assert GitEntry.spec_for(mirrors['mirrored_path'], 'refs/heads/branch') == {
'sweet': ('totally!', 0644)
}
footers = mirrors['mirrored_path']['refs/heads/branch'].commit.data.footers
assert thaw(footers) == OrderedDict([
('Commit-Id', ['whaaat']),
('Cr-Original-Commit-Position', ['refs/heads/branch@{#12345}']),
('Cr-Original-Branched-From', ['deadbeef-refs/heads/master@{#12300}']),
('Cr-Mirrored-From', ['[FILE-URL]']),
('Cr-Mirrored-Commit', ['b404e807c89d3b8f4b255fec1aaa9e123808f63c']),
])
def halt_on_bad_mirror_commit(origin, run, checkpoint, mirrors,
config, local_origin_repo, **_):
master = origin['refs/heads/master']
master.make_commit('initial commit', {'mirrored_path': {'file': 'data'}})
checkpoint('a single commit')
run()
checkpoint('a single mirrored commit')
footers = mirrors['mirrored_path']['refs/heads/master'].commit.data.footers
assert thaw(footers) == OrderedDict([
('Cr-Mirrored-From', ['[FILE-URL]']),
('Cr-Mirrored-Commit', ['7002d44b73ea8a85ee2b3e8f5f81c8c5d2ff557a']),
]), footers
mhead = mirrors['mirrored_path']['refs/heads/master']
mhead.update_to(mhead.commit.alter(
footers={
'Cr-Mirrored-Commit': ['deadbeefdeadbeefdeadbeefdeadbeefdeadbeef']
}
))
# force a resync of gsubtreed's local copy of the mirrored_path subtree repo.
base_url = config['base_url']
subtree_repo = repo.Repo(posixpath.join(base_url, 'mirrored_path'))
subtree_repo.repos_dir = local_origin_repo.repos_dir
subtree_repo.reify(share_from=local_origin_repo)
shutil.rmtree(subtree_repo.repo_path)
checkpoint('altered mirrored commit')
run()
checkpoint('should have bonked out')
def halt_on_bad_mirror_commit(origin, run, checkpoint, mirrors, config,
local_origin_repo, **_):
master = origin['refs/heads/master']
master.make_commit('initial commit', {'mirrored_path': {'file': 'data'}})
checkpoint('a single commit')
run()
checkpoint('a single mirrored commit')
footers = mirrors['mirrored_path']['refs/heads/master'].commit.data.footers
assert thaw(footers) == OrderedDict([
('Cr-Mirrored-From', ['[FILE-URL]']),
('Cr-Mirrored-Commit', ['7002d44b73ea8a85ee2b3e8f5f81c8c5d2ff557a']),
]), footers
mhead = mirrors['mirrored_path']['refs/heads/master']
mhead.update_to(
mhead.commit.alter(footers={
'Cr-Mirrored-Commit': ['deadbeefdeadbeefdeadbeefdeadbeefdeadbeef']
}))
# force a resync of gsubtreed's local copy of the mirrored_path subtree repo.
base_url = config['base_url']
subtree_repo = repo.Repo(posixpath.join(base_url, 'mirrored_path'))
subtree_repo.repos_dir = local_origin_repo.repos_dir
subtree_repo.reify(share_from=local_origin_repo)
shutil.rmtree(subtree_repo.repo_path)
checkpoint('altered mirrored commit')
run()
checkpoint('should have bonked out')
def fix_footers(origin, run, checkpoint, mirrors, **_):
branch = origin['refs/heads/branch']
branch.make_commit(
'sweet commit', {'mirrored_path': {
'sweet': 'totally!'
}},
OrderedDict([
('git-svn-id', ['totally annoying!']),
('Cr-Commit-Position', ['refs/heads/branch@{#12345}']),
('Commit-Id', ['whaaat']),
('Cr-Branched-From', ['deadbeef-refs/heads/master@{#12300}']),
]))
checkpoint('a really sweet commit')
run()
checkpoint('a really sweet (mirrored) commit')
assert GitEntry.spec_for(mirrors['mirrored_path'],
'refs/heads/branch') == {
'sweet': ('totally!', 0644)
}
footers = mirrors['mirrored_path']['refs/heads/branch'].commit.data.footers
assert thaw(footers) == OrderedDict([
('Commit-Id', ['whaaat']),
('Cr-Original-Commit-Position', ['refs/heads/branch@{#12345}']),
('Cr-Original-Branched-From', ['deadbeef-refs/heads/master@{#12300}']),
('Cr-Mirrored-From', ['[FILE-URL]']),
('Cr-Mirrored-Commit', ['b404e807c89d3b8f4b255fec1aaa9e123808f63c']),
])
def content_of(commit):
"""Calculates the content of ``commit`` such that a gnumbd-landed commit and
the original commit will compare as equals. Returns the content as a
git2.CommitData object.
This strips out:
* The parent(s)
* The committer date
* footers beginning with 'Cr-'
* the 'git-svn-id' footer.
* the '(cherry picked from ...)' line.
Stores a cached copy of the result data on the ``commit`` instance itself.
"""
if commit is None:
return git2.INVALID
if not hasattr(commit, '_cr_content'):
d = commit.data
d = tweak_cherry_pick(d)
footers = infra_types.thaw(d.footers)
footers[GIT_SVN_ID] = None
for k in footers.keys():
if k.startswith(FOOTER_PREFIX):
footers[k] = None
commit._cr_content = d.alter(
parents=(),
committer=d.committer.alter(timestamp=git2.data.NULL_TIMESTAMP),
footers=footers)
return commit._cr_content # pylint: disable=W0212
def testSet(self):
s = {1, 2, 'cat'}
f = infra_types.freeze(s)
t = infra_types.thaw(f)
self.assertEqual(s, f)
self.assertEqual(f, t)
self.assertEqual(t, s)
self.assertIsInstance(t, set)
def testList(self):
l = [1, 2, {'bob': 100}]
f = infra_types.freeze(l)
t = infra_types.thaw(f)
self.assertSequenceEqual(l, f)
self.assertSequenceEqual(f, t)
self.assertSequenceEqual(l, t)
self.assertIsInstance(t, list)
def testSet(self):
s = {1, 2, 'cat'}
f = infra_types.freeze(s)
t = infra_types.thaw(f)
self.assertEqual(s, f)
self.assertEqual(f, t)
self.assertEqual(t, s)
self.assertIsInstance(t, set)
def testList(self):
l = [1, 2, {'bob': 100}]
f = infra_types.freeze(l)
t = infra_types.thaw(f)
self.assertSequenceEqual(l, f)
self.assertSequenceEqual(f, t)
self.assertSequenceEqual(l, t)
self.assertIsInstance(t, list)
def testOrderedDictRetainsOrder(self): d = collections.OrderedDict() d['cat'] = 100 d['dog'] = 0 f = infra_types.freeze(d) t = infra_types.thaw(f) self.assertEqual(d, f) self.assertEqual(t, f) self.assertEqual(d, t) self.assertIsInstance(t, collections.OrderedDict)
def testOrderedDictRetainsOrder(self):
d = collections.OrderedDict()
d['cat'] = 100
d['dog'] = 0
f = infra_types.freeze(d)
t = infra_types.thaw(f)
self.assertEqual(d, f)
self.assertEqual(t, f)
self.assertEqual(d, t)
self.assertIsInstance(t, collections.OrderedDict)
def testDict(self):
d = {
'cat': 100,
'dog': 0,
}
f = infra_types.freeze(d)
t = infra_types.thaw(f)
self.assertEqual(d, f)
self.assertEqual(t, f)
self.assertEqual(d, t)
self.assertIsInstance(t, collections.OrderedDict)
def testDict(self):
d = {
'cat': 100,
'dog': 0,
}
f = infra_types.freeze(d)
t = infra_types.thaw(f)
self.assertEqual(d, f)
self.assertEqual(t, f)
self.assertEqual(d, t)
self.assertIsInstance(t, collections.OrderedDict)
def run(include_log=True):
stdout = sys.stdout
stderr = sys.stderr
if include_log:
logout = StringIO()
root_logger = logging.getLogger()
log_level = root_logger.getEffectiveLevel()
shandler = logging.StreamHandler(logout)
shandler.setFormatter(
logging.Formatter('%(levelname)s: %(message)s'))
root_logger.addHandler(shandler)
root_logger.setLevel(logging.INFO)
success = False
synthesized_commits = []
try:
sys.stderr = sys.stdout = open(os.devnull, 'w')
local.reify()
success, synthesized_commits = gnumbd.inner_loop(local, cref, clock)
except Exception: # pragma: no cover
import traceback
ret.append(traceback.format_exc().splitlines())
finally:
sys.stdout = stdout
sys.stderr = stderr
if include_log:
root_logger.removeHandler(shandler)
root_logger.setLevel(log_level)
# infra.libs.git2.repo logs this message if the command took longer than
# 1s to run. This causes test flakes occasionally.
log_lines = [x for x in logout.getvalue().splitlines()
if 'Finished in ' not in x]
ret.append({'log output': log_lines})
ret.append({
'inner_loop success': success,
'synthesized_commits': [
{
'commit': c.hsh,
'footers': infra_types.thaw(c.data.footers),
} for c in synthesized_commits
],
})
def run(include_log=True):
stdout = sys.stdout
stderr = sys.stderr
if include_log:
logout = StringIO()
root_logger = logging.getLogger()
log_level = root_logger.getEffectiveLevel()
shandler = logging.StreamHandler(logout)
shandler.setFormatter(
logging.Formatter('%(levelname)s: %(message)s'))
root_logger.addHandler(shandler)
root_logger.setLevel(logging.INFO)
success = False
synthesized_commits = []
try:
sys.stderr = sys.stdout = open(os.devnull, 'w')
local.reify()
success, synthesized_commits = gnumbd.inner_loop(
local, cref, clock)
except Exception: # pragma: no cover
import traceback
ret.append(traceback.format_exc().splitlines())
finally:
sys.stdout = stdout
sys.stderr = stderr
if include_log:
root_logger.removeHandler(shandler)
root_logger.setLevel(log_level)
ret.append({'log output': logout.getvalue().splitlines()})
ret.append({
'inner_loop success':
success,
'synthesized_commits': [{
'commit':
c.hsh,
'footers':
infra_types.thaw(c.data.footers),
} for c in synthesized_commits],
})
def main(args): # pragma: no cover
opts = parse_args(args)
commits_counter = ts_mon.CounterMetric('gnumbd/commit_count')
cref = gnumbd.GnumbdConfigRef(opts.repo)
opts.repo.reify()
all_commits = []
def outer_loop_iteration():
success, commits = gnumbd.inner_loop(opts.repo, cref)
all_commits.extend(commits)
commits_counter.increment_by(len(commits))
return success
# TODO(iannucci): sleep_timeout should be an exponential backon/off.
# Whenever we push, we should decrease the interval at 'backon_rate'
# until we hit 'min_interval'.
# Whenever we fail/NOP, we should back off at 'backoff_rate' until we
# hit 'max_interval'.
#
# When all is going well, this should be looping at < 1 sec. If things
# start going sideways, we should automatically back off.
loop_results = outer_loop.loop(task=outer_loop_iteration,
sleep_timeout=lambda: cref['interval'],
**opts.loop_opts)
if opts.json_output:
with open(opts.json_output, 'w') as f:
json.dump(
{
'error_count':
loop_results.error_count,
'synthesized_commits':
[{
'commit': c.hsh,
'footers': infra_types.thaw(c.data.footers),
} for c in all_commits],
}, f)
return 0 if loop_results.success else 1
def main(args): # pragma: no cover
opts = parse_args(args)
commits_counter = ts_mon.CounterMetric('gnumbd/commit_count')
cref = gnumbd.GnumbdConfigRef(opts.repo)
opts.repo.reify()
all_commits = []
def outer_loop_iteration():
success, commits = gnumbd.inner_loop(opts.repo, cref)
all_commits.extend(commits)
commits_counter.increment_by(len(commits))
return success
# TODO(iannucci): sleep_timeout should be an exponential backon/off.
# Whenever we push, we should decrease the interval at 'backon_rate'
# until we hit 'min_interval'.
# Whenever we fail/NOP, we should back off at 'backoff_rate' until we
# hit 'max_interval'.
#
# When all is going well, this should be looping at < 1 sec. If things
# start going sideways, we should automatically back off.
loop_results = outer_loop.loop(
task=outer_loop_iteration,
sleep_timeout=lambda: cref['interval'],
**opts.loop_opts)
if opts.json_output:
with open(opts.json_output, 'w') as f:
json.dump({
'error_count': loop_results.error_count,
'synthesized_commits': [
{
'commit': c.hsh,
'footers': infra_types.thaw(c.data.footers),
} for c in all_commits
],
}, f)
return 0 if loop_results.success else 1
