def test_invalid_compare(self):
with self.assertRaises(ValueError):
Commit(revision=1).__cmp__(Commit(identifier=2))
with self.assertRaises(ValueError):
Commit(revision=1,
timestamp=100).__cmp__(Commit(identifier=2, timestamp=100))
def test_branch_point(self):
self.assertEqual('0', str(Commit(identifier=0)))
self.assertEqual('0@branch-a',
str(Commit(identifier=0, branch='branch-a')))
self.assertEqual(
'1234.0@branch-a',
str(Commit(branch_point=1234, identifier=0, branch='branch-a')))
def test_siblings(self, client, **kwargs):
self.register_all_commits(client)
response = client.get(
self.URL + '/api/commits/siblings?repository_id=webkit&id=6')
self.assertEqual(200, response.status_code)
commits = {
key: [Commit.from_json(element).hash for element in values]
for key, values in response.json().items()
}
self.assertEqual(
commits,
dict(safari=[
'd8bce26fa65c6fc8f39c17927abb77f69fab82fc',
'bae5d1e90999d4f916a8a15810ccfa43f37a2fd6',
'1abe25b443e985f93b90d830e4a7e3731336af4d',
'fff83bb2d9171b4d9196e977eb0508fd57e7a08d',
'9b8311f25a77ba14923d9d5a6532103f54abefcb',
]),
)
response = client.get(
self.URL +
'/api/commits/siblings?repository_id=safari&id=d8bce26fa65c')
self.assertEqual(200, response.status_code)
commits = {
key: [Commit.from_json(element).revision for element in values]
for key, values in response.json().items()
}
self.assertEqual(commits, dict(webkit=[6]))
def test_from_json(self):
self.assertEqual(
Commit.from_json(
dict(
repository_id='webkit',
id=1234,
timestamp=1000,
)),
Commit(
repository_id='webkit',
revision=1234,
timestamp=1000,
),
)
self.assertEqual(
Commit.from_json(
dict(
repository_id='webkit',
id='c3bd784f8b88bd03f64467ddd3304ed8be28acbe',
timestamp=2000,
)),
Commit(
repository_id='webkit',
hash='c3bd784f8b88bd03f64467ddd3304ed8be28acbe',
timestamp=2000,
),
)
def test_commits(self):
with mocks.local.Svn(self.path), OutputCapture():
svn = local.Svn(self.path)
self.assertEqual(Commit.Encoder().default([
svn.commit(revision='r6'),
svn.commit(revision='r4'),
svn.commit(revision='r2'),
svn.commit(revision='r1'),
]), Commit.Encoder().default(list(svn.commits(begin=dict(revision='r1'), end=dict(revision='r6')))))
def test_commits_branch(self):
with mocks.remote.Svn(), OutputCapture():
svn = remote.Svn(self.remote)
self.assertEqual(Commit.Encoder().default([
svn.commit(revision='r7'),
svn.commit(revision='r3'),
svn.commit(revision='r2'),
svn.commit(revision='r1'),
]), Commit.Encoder().default(list(svn.commits(begin=dict(argument='r1'), end=dict(argument='r7')))))
def test_branch_commits(self):
with OutputCapture() as captured, mocks.local.Git(
self.path) as mock, mocks.local.Svn(), MockTime:
contirbutors = Contributor.Mapping()
contirbutors.create('Jonathan Bedard', '*****@*****.**')
local.Git(self.path).checkout('branch-a')
mock.commits['branch-a'].append(
Commit(
hash='f93138e3bf1d5ecca25fc0844b7a2a78b8e00aae',
branch='branch-a',
author=Contributor('*****@*****.**',
emails=['*****@*****.**']),
branch_point=mock.commits['branch-a'][-1].branch_point,
identifier=mock.commits['branch-a'][-1].identifier + 1,
timestamp=1601668000,
message='New commit 1\n',
))
mock.commits['branch-a'].append(
Commit(
hash='0148c0df0faf248aa133d6d5ad911d7cb1b56a5b',
branch='branch-a',
author=Contributor('*****@*****.**',
emails=['*****@*****.**']),
branch_point=mock.commits['branch-a'][-1].branch_point,
identifier=mock.commits['branch-a'][-1].identifier + 1,
timestamp=1601669000,
message='New commit 2\n',
))
self.assertEqual(
0,
program.main(
args=('canonicalize', ),
path=self.path,
contributors=contirbutors,
))
commit_a = local.Git(self.path).commit(branch='branch-a~1')
self.assertEqual(commit_a.author,
contirbutors['*****@*****.**'])
self.assertEqual(commit_a.message,
'New commit 1\nIdentifier: 2.3@branch-a')
commit_b = local.Git(self.path).commit(branch='branch-a')
self.assertEqual(commit_b.author,
contirbutors['*****@*****.**'])
self.assertEqual(commit_b.message,
'New commit 2\nIdentifier: 2.4@branch-a')
self.assertEqual(
captured.stdout.getvalue(),
'Rewrite f93138e3bf1d5ecca25fc0844b7a2a78b8e00aae (1/2) (--- seconds passed, remaining --- predicted)\n'
'Rewrite 0148c0df0faf248aa133d6d5ad911d7cb1b56a5b (2/2) (--- seconds passed, remaining --- predicted)\n'
'2 commits successfully canonicalized!\n',
)
def test_commits(self):
self.maxDiff = None
with mocks.remote.Svn():
svn = remote.Svn(self.remote)
self.assertEqual(Commit.Encoder().default([
svn.commit(revision='r6'),
svn.commit(revision='r4'),
svn.commit(revision='r2'),
svn.commit(revision='r1'),
]), Commit.Encoder().default(list(svn.commits(begin=dict(revision='r1'), end=dict(revision='r6')))))
def test_contributor(self):
contributor = Contributor.from_scm_log(
'Author: Jonathan Bedard <*****@*****.**>')
commit = Commit(revision=1, identifier=1, author=contributor)
self.assertEqual(commit.author, contributor)
commit = Commit(revision=1,
identifier=1,
author=Contributor.Encoder().default(contributor))
self.assertEqual(commit.author, contributor)
def test_parse_hash(self):
self.assertEqual(
'1a2e41e3f7cdf51b1e1d02880cfb65eab9327ef2',
Commit._parse_hash('1a2e41e3f7cdf51b1e1d02880cfb65eab9327ef2'))
self.assertEqual(
'c3bd784f8b88bd03f64467ddd3304ed8be28acbe',
Commit._parse_hash('C3BD784F8B88BD03F64467DDD3304ED8BE28ACBE'))
self.assertEqual(None, Commit._parse_hash('invalid hash'))
self.assertEqual(
None,
Commit._parse_hash('c3bd784f8b88bd03f64467ddd3304ed8be28acbe1'))
def test_next(self, client, **kwargs):
self.register_all_commits(client)
response = client.get(self.URL + '/api/commits/next?id=bae5d1e90999')
self.assertEqual(200, response.status_code)
self.assertEqual(1, len(response.json()))
self.assertEqual('d8bce26fa65c6fc8f39c17927abb77f69fab82fc',
Commit.from_json(response.json()[0]).hash)
response = client.get(self.URL + '/api/commits/next?id=4')
self.assertEqual(200, response.status_code)
self.assertEqual(1, len(response.json()))
self.assertEqual(6, Commit.from_json(response.json()[0]).revision)
def test_find_uuid(self, client, **kwargs):
self.register_all_commits(client)
response = client.get(self.URL + '/api/commits?uuid=160166800001')
self.assertEqual(200, response.status_code)
self.assertEqual(1, len(response.json()))
self.assertEqual(
Commit.from_json(response.json()[0]).hash,
'd8bce26fa65c6fc8f39c17927abb77f69fab82fc')
response = client.get(self.URL + '/api/commits?uuid=160163990000')
self.assertEqual(200, response.status_code)
self.assertEqual(1, len(response.json()))
self.assertEqual(Commit.from_json(response.json()[0]).revision, 6)
def test_find_range_id(self, client, **kwargs):
self.register_all_commits(client)
response = client.get(self.URL +
'/api/commits?after_id=6&before_id=fff83bb2d917')
self.assertEqual(200, response.status_code)
self.assertEqual([
Commit.from_json(element).revision
or Commit.from_json(element).hash for element in response.json()
], [
6,
'9b8311f25a77ba14923d9d5a6532103f54abefcb',
'fff83bb2d9171b4d9196e977eb0508fd57e7a08d',
])
def test_repr(self):
self.assertEqual(str(Commit(identifier=123, branch='main')),
'123@main')
self.assertEqual(
str(Commit(branch_point=1234, identifier=1, branch='eng/1234')),
'1234.1@eng/1234')
self.assertEqual(str(Commit(identifier=123)), '123')
self.assertEqual(str(Commit(revision=123)), 'r123')
self.assertEqual(
str(Commit(hash='c3bd784f8b88bd03f64467ddd3304ed8be28acbe')),
'c3bd784f8b88')
def test_previous(self, client, **kwargs):
self.register_all_commits(client)
response = client.get(self.URL +
'/api/commits/previous?id=d8bce26fa65c')
self.assertEqual(200, response.status_code)
self.assertEqual(1, len(response.json()))
self.assertEqual('bae5d1e90999d4f916a8a15810ccfa43f37a2fd6',
Commit.from_json(response.json()[0]).hash)
response = client.get(self.URL + '/api/commits/previous?id=6')
self.assertEqual(200, response.status_code)
self.assertEqual(1, len(response.json()))
self.assertEqual(4, Commit.from_json(response.json()[0]).revision)
def test_commits_branch(self):
with mocks.remote.GitHub():
git = remote.GitHub(self.remote)
self.assertEqual(
Commit.Encoder().default([
git.commit(hash='621652ad'),
git.commit(hash='a30ce849'),
git.commit(hash='fff83bb2'),
git.commit(hash='9b8311f2'),
]),
Commit.Encoder().default(
list(
git.commits(begin=dict(argument='9b8311f2'),
end=dict(argument='621652ad')))))
def test_commits(self):
with mocks.remote.GitHub():
git = remote.GitHub(self.remote)
self.assertEqual(
Commit.Encoder().default([
git.commit(hash='bae5d1e9'),
git.commit(hash='1abe25b4'),
git.commit(hash='fff83bb2'),
git.commit(hash='9b8311f2'),
]),
Commit.Encoder().default(
list(
git.commits(begin=dict(hash='9b8311f2'),
end=dict(hash='bae5d1e9')))))
def test_parse_revision(self):
self.assertEqual(266896, Commit._parse_revision('r266896'))
self.assertEqual(266896, Commit._parse_revision('R266896'))
self.assertEqual(266896, Commit._parse_revision('266896'))
self.assertEqual(266896, Commit._parse_revision(266896))
self.assertEqual(None, Commit._parse_revision('c3bd784f8b88bd03'))
self.assertEqual(None, Commit._parse_revision('0'))
self.assertEqual(None, Commit._parse_revision('-1'))
self.assertEqual(None, Commit._parse_revision('3.141592'))
self.assertEqual(None, Commit._parse_revision(3.141592))
def test_find_timestamp(self, client, **kwargs):
self.register_all_commits(client)
response = client.get(self.URL + '/api/commits?timestamp=1601668000')
self.assertEqual(200, response.status_code)
self.assertEqual(2, len(response.json()))
self.assertEqual(
[Commit.from_json(element).hash for element in response.json()], [
'bae5d1e90999d4f916a8a15810ccfa43f37a2fd6',
'd8bce26fa65c6fc8f39c17927abb77f69fab82fc',
])
response = client.get(self.URL + '/api/commits?timestamp=1601639900')
self.assertEqual(200, response.status_code)
self.assertEqual(1, len(response.json()))
self.assertEqual(Commit.from_json(response.json()[0]).revision, 6)
def test_json_decode(self):
contributor = Contributor.from_scm_log(
'Author: Jonathan Bedard <*****@*****.**>')
commit_a = Commit(revision=1,
hash='c3bd784f8b88bd03f64467ddd3304ed8be28acbe',
identifier='1@main',
timestamp=1000,
author=Contributor.Encoder().default(contributor),
message='Message')
dictionary = json.loads(json.dumps(commit_a, cls=Commit.Encoder))
commit_b = Commit(**dictionary)
self.assertEqual(commit_a, commit_b)
def checkout(self, argument):
if not isinstance(argument, six.string_types):
raise ValueError(
"Expected 'argument' to be a string, not '{}'".format(
type(argument)))
self._branch = None
if log.level > logging.WARNING:
log_arg = ['-q']
elif log.level < logging.WARNING:
log_arg = ['--progress']
else:
log_arg = []
parsed_commit = Commit.parse(argument, do_assert=False)
if parsed_commit:
commit = self.commit(
hash=parsed_commit.hash,
revision=parsed_commit.revision,
identifier=parsed_commit.identifier,
branch=parsed_commit.branch,
)
return None if run(
[self.executable(), 'checkout'] + [commit.hash] + log_arg,
cwd=self.root_path,
).returncode else commit
return None if run(
[self.executable(), 'checkout'] + [argument] + log_arg,
cwd=self.root_path,
).returncode else self.commit()
def __init__(
self,
remote='github.example.com/WebKit/webkit',
datafile=None,
default_branch='main',
git_svn=False,
):
if not scmremote.GitHub.is_webserver('https://{}'.format(remote)):
raise ValueError(
'"{}" is not a valid GitHub remote'.format(remote))
self.default_branch = default_branch
self.remote = remote
hostname = self.remote.split('/')[0]
self.api_remote = 'api.{hostname}/repos/{repo}'.format(
hostname=hostname,
repo='/'.join(self.remote.split('/')[1:]),
)
super(GitHub, self).__init__(hostname, 'api.{}'.format(hostname))
with open(datafile
or os.path.join(os.path.dirname(os.path.dirname(__file__)),
'git-repo.json')) as file:
self.commits = json.load(file)
for key, commits in self.commits.items():
self.commits[key] = [Commit(**kwargs) for kwargs in commits]
if not git_svn:
for commit in self.commits[key]:
commit.revision = None
self.head = self.commits[self.default_branch][-1]
self.tags = {}
self._environment = None
def find(self, argument, include_log=True, include_identifier=True):
if not isinstance(argument, six.string_types):
raise ValueError("Expected 'argument' to be a string, not '{}'".format(type(argument)))
if argument in self.DEFAULT_BRANCHES:
argument = self.default_branch
parsed_commit = Commit.parse(argument, do_assert=False)
if parsed_commit:
if parsed_commit.branch in self.DEFAULT_BRANCHES:
parsed_commit.branch = self.default_branch
return self.commit(
hash=parsed_commit.hash,
revision=parsed_commit.revision,
identifier=parsed_commit.identifier,
branch=parsed_commit.branch,
include_log=include_log,
include_identifier=include_identifier,
)
commit_data = self.request('commits/{}'.format(argument))
if not commit_data:
raise ValueError("'{}' is not an argument recognized by git".format(argument))
return self.commit(hash=commit_data['sha'], include_log=include_log, include_identifier=include_identifier)
def test_json_encode(self):
contributor = Contributor.from_scm_log(
'Author: Jonathan Bedard <*****@*****.**>')
self.assertDictEqual(
dict(
revision=1,
hash='c3bd784f8b88bd03f64467ddd3304ed8be28acbe',
branch='main',
identifier='1@main',
timestamp=1000,
order=0,
author=dict(
name='Jonathan Bedard',
emails=['*****@*****.**'],
),
message='Message',
),
json.loads(
json.dumps(Commit(
revision=1,
hash='c3bd784f8b88bd03f64467ddd3304ed8be28acbe',
identifier='1@main',
timestamp=1000,
author=contributor,
message='Message',
),
cls=Commit.Encoder)))
def _do_job_for_key(self, key, attempts=1):
job_complete = False
try:
raw_data = self.redis.get(f'data_for_{key}')
if raw_data:
data = json.loads(raw_data)
self.synchronously_process_test_results(
configuration=Configuration.from_json(
data['configuration']),
commits=[
Commit.from_json(commit_json)
for commit_json in data['commits']
],
suite=data['suite'],
timestamp=data['timestamp'],
test_results=data['test_results'],
)
job_complete = True
finally:
if job_complete or attempts >= self.MAX_ATTEMPTS:
self.redis.delete(key)
self.redis.delete(f'data_for_{key}')
else:
self.redis.set(
key,
json.dumps(dict(started_processing=0, attempts=attempts)),
ex=self.PROCESS_TIMEOUT,
)
return job_complete
def webkit(cls):
svn = mocks.remote.Svn(remote='svn.webkit.org/repository/webkit')
github = mocks.remote.GitHub(remote='github.com/WebKit/WebKit',
git_svn=True)
github.commits = {}
for branch, commits in svn.commits.items():
if 'tag' in branch:
continue
if branch == 'trunk':
branch = 'main'
github.commits[branch] = []
for commit in commits:
github.commits[branch].append(
Commit(
repository_id=commit.repository_id,
branch=branch,
author=commit.author,
message=commit.message,
timestamp=commit.timestamp,
order=commit.order,
identifier=commit.identifier,
branch_point=commit.branch_point,
revision=commit.revision,
hash=commit.hash,
))
with svn, github:
yield github
def info(self, branch=None, revision=None, tag=None):
if tag and branch:
raise ValueError('Cannot specify both branch and tag')
if tag and revision:
raise ValueError('Cannot specify both branch and tag')
revision = Commit._parse_revision(revision)
if branch and branch != self.default_branch and '/' not in branch:
branch = 'branches/{}'.format(branch)
additional_args = [
'^/{}'.format(branch)
] if branch and branch != self.default_branch else []
additional_args += ['^/tags/{}'.format(tag)] if tag else []
additional_args += ['-r', str(revision)] if revision else []
info_result = run([self.executable(), 'info'] + additional_args,
cwd=self.root_path,
capture_output=True,
encoding='utf-8')
if info_result.returncode:
return {}
result = {}
for line in info_result.stdout.splitlines():
split = line.split(': ')
result[split[0]] = ': '.join(split[1:])
return result
def __init__(
self,
remote='bitbucket.example.com/projects/WEBKIT/repos/webkit',
datafile=None,
default_branch='main',
git_svn=False,
):
if not scmremote.BitBucket.is_webserver('https://{}'.format(remote)):
raise ValueError(
'"{}" is not a valid BitBucket remote'.format(remote))
self.default_branch = default_branch
self.remote = remote
self.project = '/'.join(remote.split('/')[1:])
super(BitBucket, self).__init__(self.remote.split('/')[0])
with open(datafile
or os.path.join(os.path.dirname(os.path.dirname(__file__)),
'git-repo.json')) as file:
self.commits = json.load(file)
for key, commits in self.commits.items():
self.commits[key] = [Commit(**kwargs) for kwargs in commits]
if not git_svn:
for commit in self.commits[key]:
commit.revision = None
self.head = self.commits[self.default_branch][-1]
self.tags = {}
def find(self, argument, include_log=True):
if not isinstance(argument, six.string_types):
raise ValueError(
"Expected 'argument' to be a string, not '{}'".format(
type(argument)))
parsed_commit = Commit.parse(argument, do_assert=False)
if parsed_commit:
return self.commit(
hash=parsed_commit.hash,
revision=parsed_commit.revision,
identifier=parsed_commit.identifier,
branch=parsed_commit.branch,
include_log=include_log,
)
output = run(
[self.executable(), 'rev-parse', argument],
cwd=self.root_path,
capture_output=True,
encoding='utf-8',
)
if output.returncode:
raise ValueError(
"'{}' is not an argument recognized by git".format(argument))
return self.commit(hash=output.stdout.rstrip(),
include_log=include_log)
def commits(self, begin=None, end=None, include_log=True, include_identifier=True):
begin, end = self._commit_range(begin=begin, end=end, include_identifier=include_identifier)
previous = end
cached = [previous]
while previous:
response = self.request('commits', paginate=False, params=dict(sha=previous.hash))
if not response:
break
for commit_data in response:
branch_point = previous.branch_point
identifier = previous.identifier
if commit_data['sha'] == previous.hash:
cached = cached[:-1]
else:
identifier -= 1
if not identifier:
identifier = branch_point
branch_point = None
matches = self.GIT_SVN_REVISION.findall(commit_data['commit']['message'])
revision = int(matches[-1].split('@')[0]) if matches else None
email_match = self.EMAIL_RE.match(commit_data['commit']['author']['email'])
timestamp = int(calendar.timegm(datetime.strptime(
commit_data['commit']['committer']['date'], '%Y-%m-%dT%H:%M:%SZ',
).timetuple()))
previous = Commit(
repository_id=self.id,
hash=commit_data['sha'],
revision=revision,
branch=end.branch if identifier and branch_point else self.default_branch,
identifier=identifier if include_identifier else None,
branch_point=branch_point if include_identifier else None,
timestamp=timestamp,
author=self.contributors.create(
commit_data['commit']['author']['name'],
email_match.group('email') if email_match else None,
), order=0,
message=commit_data['commit']['message'] if include_log else None,
)
if not cached or cached[0].timestamp != previous.timestamp:
for c in cached:
yield c
cached = [previous]
else:
for c in cached:
c.order += 1
cached.append(previous)
if previous.hash == begin.hash or previous.timestamp < begin.timestamp:
previous = None
break
for c in cached:
c.order += begin.order
yield c