def register_all_commits(self, client):
for repo in [MockStashRepository.safari(), MockSVNRepository.webkit()]:
for commits in repo.commits.values():
for commit in commits:
self.assertEqual(
200,
client.post(
self.URL + '/api/commits/register',
data=Commit.Encoder().default(commit)).status_code)
def test_udid(self):
self.assertEqual(
Commit(
repository_id='safari',
branch='master',
id='7be4084258a452e8fe22f36287c5b321e9c8249b',
timestamp=1537550685,
order=1,
).uuid, 153755068501)
def to_commit(self):
return Commit(
repository_id=self.repository_id,
branch=self.branch,
id=self.commit_id,
timestamp=self.uuid // Commit.TIMESTAMP_TO_UUID_MULTIPLIER,
order=self.uuid % Commit.TIMESTAMP_TO_UUID_MULTIPLIER,
committer=self.committer,
message=self.message,
)
def test_register_with_full_commit(self, client, **kwargs):
git_commit = MockStashRepository.safari().commit_for_id(
'bb6bda5f44dd2')
self.assertEqual(
200,
client.post(self.URL + '/api/commits',
data=Commit.Encoder().default(git_commit)).status_code)
response = client.get(self.URL + '/api/commits?repository_id=safari')
self.assertEqual(200, response.status_code)
self.assertEqual(Commit.from_json(response.json()[0]), git_commit)
svn_commit = MockSVNRepository.webkit().commit_for_id('236544')
self.assertEqual(
200,
client.post(self.URL + '/api/commits',
data=Commit.Encoder().default(svn_commit)).status_code)
response = client.get(self.URL + '/api/commits?repository_id=webkit')
self.assertEqual(200, response.status_code)
self.assertEqual(Commit.from_json(response.json()[0]), svn_commit)
def test_find_range_id(self, client, **kwargs):
self.register_all_commits(client)
response = client.get(
self.URL + '/api/commits?after_id=336610a8&before_id=236540')
self.assertEqual(200, response.status_code)
self.assertEqual(
[Commit.from_json(element) for element in response.json()], [
MockStashRepository.safari().commit_for_id(id='336610a8'),
MockStashRepository.safari().commit_for_id(id='bb6bda5f'),
MockSVNRepository.webkit().commit_for_id(id=236540),
])
def test_find_range_id(self, client, **kwargs):
self.register_all_commits(client)
response = client.get(self.URL +
'/api/commits?after_id=6&before_id=fff83bb2d917')
self.assertEqual(200, response.status_code)
self.assertEqual(
[Commit.from_json(element).id for element in response.json()], [
'6',
'9b8311f25a77ba14923d9d5a6532103f54abefcb',
'fff83bb2d9171b4d9196e977eb0508fd57e7a08d',
])
def commit_for_id(self, id):
commit = self.remote.commit(revision=id, include_identifier=False)
return Commit(
repository_id=self.key,
id=commit.revision,
branch=commit.branch,
timestamp=commit.timestamp,
order=commit.order,
committer=commit.author.email,
message=commit.message,
)
def test_compare(self):
commit1 = Commit(
repository_id='safari',
branch='master',
id='e64810a40c3fecb728871e12ca31482ca715b383',
timestamp=1537550685,
)
commit2 = Commit(
repository_id='safari',
branch='master',
id='7be4084258a452e8fe22f36287c5b321e9c8249b',
timestamp=1537550685,
order=1,
)
commit3 = Commit(
repository_id='safari',
branch='master',
id='bb6bda5f44dd24d0b54539b8ff6e8c17f519249a',
timestamp=1537810281,
)
commit4 = Commit(
repository_id='webkit',
branch='master',
id=236522,
timestamp=1537826614,
)
self.assertTrue(commit2 > commit1)
self.assertTrue(commit2 >= commit1)
self.assertTrue(commit1 >= commit1)
self.assertTrue(commit1 < commit2)
self.assertTrue(commit1 <= commit2)
self.assertTrue(commit1 <= commit1)
self.assertTrue(commit1 == commit1)
self.assertTrue(commit2 < commit3)
self.assertTrue(commit3 > commit2)
self.assertEqual(commit1.timestamp, commit2.timestamp)
self.assertTrue(commit4 > commit3)
self.assertNotEqual(commit3.repository_id, commit4.repository_id)
def unpack(self):
return dict(
commits=[
Commit.from_json(element) for element in json.loads(
UploadContext.from_zip(bytearray(self.commits)))
],
sdk=None if self.sdk == '?' else self.sdk,
test_results=json.loads(
UploadContext.from_zip(bytearray(self.test_results))),
timestamp=calendar.timegm(self.time_uploaded.timetuple()),
version=self.upload_version,
)
def run_function_through_redis_cache(self, key, function):
result = self.redis.get('commit_mapping:' + key)
if result:
try:
result = [Commit.from_json(element) for element in json.loads(result)]
except ValueError:
result = None
if result is None:
result = function()
if result:
self.redis.set('commit_mapping:' + key, json.dumps(result, cls=Commit.Encoder), ex=self.cache_timeout)
return result
def test_register_with_full_commit(self, client, **kwargs):
git_commit = Commit(
repository_id='safari',
branch='main',
id='8eba280e32daaf5fddbbb65aea37932ea9ad85df',
timestamp=1601650100,
order=0,
committer='*****@*****.**',
message='custom commit',
)
self.assertEqual(
200,
client.post(self.URL + '/api/commits',
data=Commit.Encoder().default(git_commit)).status_code)
response = client.get(self.URL + '/api/commits?repository_id=safari')
self.assertEqual(200, response.status_code)
self.assertEqual(Commit.from_json(response.json()[0]), git_commit)
svn_commit = Commit(
repository_id='webkit',
branch='trunk',
id='1234',
timestamp=1601650100,
order=0,
committer='*****@*****.**',
message='custom commit',
)
self.assertEqual(
200,
client.post(self.URL + '/api/commits',
data=Commit.Encoder().default(svn_commit)).status_code)
response = client.get(self.URL + '/api/commits?repository_id=webkit')
self.assertEqual(200, response.status_code)
self.assertEqual(Commit.from_json(response.json()[0]), svn_commit)
def _find_comparison(commit_context,
repository_id,
branch,
id,
uuid,
timestamp,
priority=min):
if bool(id) + bool(uuid) + bool(timestamp) > 1:
abort(
400,
description=
'Can only search by one of [commit id, commit uuid, timestamp] in a single request'
)
try:
if uuid:
# We don't need real commit to search by uuid and CommitContexts have trouble diffrentiating between uuids and timestamps.
uuid = priority([int(element) for element in uuid])
return Commit('?', '?', '?',
uuid // Commit.TIMESTAMP_TO_UUID_MULTIPLIER,
uuid % Commit.TIMESTAMP_TO_UUID_MULTIPLIER)
if timestamp:
return priority([round(float(element)) for element in timestamp])
except ValueError:
abort(400, description='Timestamp and uuid must be integers')
if not repository_id and not id:
return None
if not repository_id:
repository_id = commit_context.repositories.keys()
for repository in repository_id:
if repository not in commit_context.repositories.keys():
abort(
404,
description=f"\'{repository}\' is not a registered repository")
result = []
for repository in repository_id:
for b in branch:
if id:
for elm in id:
result += commit_context.find_commits_by_id(
repository, b, elm)
else:
result += commit_context.find_commits_in_range(repository,
b,
limit=1)
if not result:
abort(404,
description='No commits found matching the specified criteria')
return priority(result)
def test_find_range_uuid(self, client, **kwargs):
self.register_all_commits(client)
response = client.get(
self.URL +
'/api/commits?after_uuid=160166800000&before_uuid=160166800001')
self.assertEqual(200, response.status_code)
self.assertEqual(
[Commit.from_json(element).hash for element in response.json()],
[
'bae5d1e90999d4f916a8a15810ccfa43f37a2fd6',
'd8bce26fa65c6fc8f39c17927abb77f69fab82fc',
],
)
def test_find_range_uuid(self, client, **kwargs):
self.register_all_commits(client)
response = client.get(
self.URL +
'/api/commits?after_uuid=153755068501&before_uuid=153756638602')
self.assertEqual(200, response.status_code)
self.assertEqual(
[Commit.from_json(element) for element in response.json()],
[
MockStashRepository.safari().commit_for_id(id='7be40842'),
MockStashRepository.safari().commit_for_id(id='336610a4')
],
)
def previous(self, limit=None, **kwargs):
AssertRequest.is_type()
AssertRequest.query_kwargs_empty(limit=limit)
with self.commit_context:
commits = self._find(**kwargs)
if not commits:
abort(404, description='No commits found matching the specified criteria')
if len(commits) > 1:
abort(404, description=f'{len(commits)} commits found matching the specified criteria')
commit = self.commit_context.previous_commit(commits[0])
return jsonify(Commit.Encoder().default([commit] if commit else []))
def test_register_with_partial_commit(self, client, **kwargs):
with MockModelFactory.safari(), MockModelFactory.webkit(
), OutputCapture():
self.assertEqual(
200,
client.post(self.URL + '/api/commits',
data=dict(repository_id='safari',
id='d8bce26fa65c')).status_code)
response = client.get(self.URL +
'/api/commits?repository_id=safari')
self.assertEqual(200, response.status_code)
self.assertEqual(
Commit.from_json(response.json()[0]).message,
'Patch Series\n',
)
self.assertEqual(
404,
client.post(self.URL + '/api/commits',
data=dict(repository_id='safari',
id='aaaaaaaaaaaaa')).status_code)
self.assertEqual(
200,
client.post(self.URL + '/api/commits',
data=dict(repository_id='webkit',
id='6')).status_code)
response = client.get(self.URL +
'/api/commits?repository_id=webkit')
self.assertEqual(200, response.status_code)
self.assertEqual(
Commit.from_json(response.json()[0]).message,
'6th commit',
)
self.assertEqual(
404,
client.post(self.URL + '/api/commits',
data=dict(repository_id='webkit',
id='1234')).status_code)
def test_find_range_timestamp(self, client, **kwargs):
self.register_all_commits(client)
response = client.get(
self.URL +
'/api/commits?after_timestamp=1538041792.3&before_timestamp=1538049108'
)
self.assertEqual(200, response.status_code)
self.assertEqual(
[Commit.from_json(element) for element in response.json()],
[
MockSVNRepository.webkit().commit_for_id(id=236541),
MockSVNRepository.webkit().commit_for_id(id=236542)
],
)
def siblings(self, limit=None, **kwargs):
AssertRequest.is_type()
AssertRequest.query_kwargs_empty(limit=limit)
with self.commit_context:
commits = self._find(**kwargs)
if not commits:
abort(404, description='No commits found matching the specified criteria')
if len(commits) > 1:
abort(404, description=f'{len(commits)} commits found matching the specified criteria')
repositories = sorted(self.commit_context.repositories.keys())
repositories.remove(commits[0].repository_id)
return jsonify(Commit.Encoder().default(self.commit_context.sibling_commits(commits[0], repositories)))
def test_find_range_timestamp(self, client, **kwargs):
self.register_all_commits(client)
response = client.get(
self.URL +
'/api/commits?after_timestamp=1601637900&before_timestamp=1601639900'
)
self.assertEqual(200, response.status_code)
self.assertEqual(
[
Commit.from_json(element).revision
for element in response.json()
],
[4, 6],
)
def test_register_via_post(self, client, **kwargs):
self.assertEqual(
200,
client.post(self.URL + '/api/commits',
data=dict(repository_id='safari',
id='bb6bda5f44dd2')).status_code)
response = client.get(self.URL + '/api/commits?repository_id=safari')
self.assertEqual(200, response.status_code)
self.assertEqual(
Commit.from_json(response.json()[0]),
MockStashRepository.safari().commit_for_id('bb6bda5f44dd2'),
)
self.assertEqual(
200,
client.post(self.URL + '/api/commits',
data=dict(repository_id='webkit',
id='236544')).status_code)
response = client.get(self.URL + '/api/commits?repository_id=webkit')
self.assertEqual(200, response.status_code)
self.assertEqual(
Commit.from_json(response.json()[0]),
MockSVNRepository.webkit().commit_for_id('236544'),
)
def test_siblings(self, client, **kwargs):
self.register_all_commits(client)
response = client.get(
self.URL + '/api/commits/siblings?repository_id=webkit&id=236542')
self.assertEqual(200, response.status_code)
commits = {
key: [Commit.from_json(element) for element in values]
for key, values in response.json().items()
}
self.assertEqual(
commits, {
'safari': [
MockStashRepository.safari().commit_for_id(
id='bb6bda5f44dd24')
]
})
response = client.get(
self.URL +
'/api/commits/siblings?repository_id=safari&id=bb6bda5f44dd24')
self.assertEqual(200, response.status_code)
commits = {
key: [Commit.from_json(element) for element in values]
for key, values in response.json().items()
}
self.assertEqual(
commits, {
'webkit': [
MockSVNRepository.webkit().commit_for_id(id=236544),
MockSVNRepository.webkit().commit_for_id(id=236543),
MockSVNRepository.webkit().commit_for_id(id=236542),
MockSVNRepository.webkit().commit_for_id(id=236541),
MockSVNRepository.webkit().commit_for_id(id=236540),
]
})
def commit_for_id(self, id, branch=Repository.DEFAULT_BRANCH):
try:
with self.session():
commit_data = self.get(f'commits/{id}')
if not commit_data:
raise SCMException(
f'Commit {id} does not exist on branch {branch}')
timestamp = int(commit_data['committerTimestamp']
) // self.COMMIT_TIMESTAMP_CONVERSION
branch_filter = urllib.parse.quote(f'refs/heads/{branch}')
commits_between_commit_and_branch_head = self.get(
f"commits?since={commit_data['id']}&until={branch_filter}&limit=1",
cache=False)
if commits_between_commit_and_branch_head[
'isLastPage'] and commits_between_commit_and_branch_head[
'size'] == 0:
# We may have missed the case the commit in question is the HEAD of the branch
commits_at_branch_head = self.get(
f'commits?until={branch_filter}&limit=20', cache=False)
if all([
commit['id'] != commit_data['id']
for commit in commits_at_branch_head['values']
]):
raise SCMException(f'{id} exists, but not on {branch}')
# Ordering commits by timestamp in git is a bit problematic because multiple commits can share a timestamp.
# Generally, if your parent shares your timestamp, your order needs to be incremented.
order = 0
loop_data = commit_data
while len(loop_data['parents']) == 1 and int(
loop_data['parents'][0]['committerTimestamp']
) // self.COMMIT_TIMESTAMP_CONVERSION == timestamp:
order += 1
loop_data = self.get(
f"commits/{loop_data['parents'][0]['id']}")
return Commit(
repository_id=self.key,
id=commit_data['id'],
branch=branch,
timestamp=timestamp,
order=order,
committer=commit_data['committer']['emailAddress'],
message=commit_data['message'],
)
except ValueError:
raise SCMException(f'Failed to connect to {self.url}')
def test_basic_svn(self, redis=StrictRedis):
svn_repo = MockSVNRepository(url='svn.webkit.org/repository/webkit/',
name='webkit',
redis=redis())
svn_repo.add_commit(
Commit(
repository_id=svn_repo.name,
branch='trunk',
id=236544,
timestamp=1538052408,
order=0,
committer='*****@*****.**',
message='Change 1 description.',
))
commit = svn_repo.commit_for_id(236544, branch='trunk')
self.assertEqual(commit.uuid, 153805240800)
self.assertIsNone(commit.message, None)
def commit(self, **kwargs):
with self.commit_context:
commit = self._single_commit()
repositories = list(self.commit_context.repositories.keys())
repositories.remove(commit.repository_id)
siblings = self.commit_context.sibling_commits(
commit, repositories)
repositories = [commit.repository_id] + sorted(
[str(key) for key, lst in siblings.items() if lst])
return self.environment.get_template('commit.html').render(
title=self.site_menu.title + ': ' + str(commit.id),
commit=commit,
repository_ids=repositories,
commits=Commit.Encoder().default(
[commit] + [item for lst in siblings.values()
for item in lst]),
**kwargs)
def safari(redis=None):
result = MockStashRepository('https://fake-stash-instance.apple.com/projects/BROWSER/repos/safari', name='safari', redis=redis)
result.add_commit(Commit(
repository_id=result.name, branch='master', id='bb6bda5f44dd24d0b54539b8ff6e8c17f519249a',
timestamp=1537810281, order=0,
committer='*****@*****.**',
message='Change 1 description.',
))
result.add_commit(Commit(
repository_id=result.name, branch='master', id='336610a84fdcf14ddcf1db65075af95480516fda',
timestamp=1537809818, order=0,
committer='*****@*****.**',
message='Change 2 description.',
))
result.add_commit(Commit(
repository_id=result.name, branch='master', id='336610a40c3fecb728871e12ca31482ca715b383',
timestamp=1537566386, order=0,
committer='*****@*****.**',
message='<rdar://problem/99999999> Change 3 description.',
))
result.add_commit(Commit(
repository_id=result.name, branch='master', id='e64810a40c3fecb728871e12ca31482ca715b383',
timestamp=1537550685, order=0,
committer='*****@*****.**',
message=u'Change 4 \u2014 (Part 1) description.',
))
result.add_commit(Commit(
repository_id=result.name, branch='master', id='7be4084258a452e8fe22f36287c5b321e9c8249b',
timestamp=1537550685, order=1,
committer='*****@*****.**',
message=u'Change 4 \u2014 (Part 2) description.\nReviewed by person.',
))
result.add_commit(Commit(
repository_id=result.name, branch='safari-606-branch', id='79256c32a855ac8612112279008334d90e901c55',
timestamp=1537897367, order=0,
committer='*****@*****.**',
message='Change 5 description.',
))
result.add_commit(Commit(
repository_id=result.name, branch='safari-606-branch', id='d85222d9407fdbbf47406509400a9cecb73ac6de',
timestamp=1537563383, order=0,
committer='*****@*****.**',
message='Change 6 description.',
))
return result
def webkit(redis=None):
result = MockWebKitRepository(redis=redis)
result.add_commit(Commit(
repository_id=result.name, branch='trunk', id=236544,
timestamp=1538052408, order=0,
committer='*****@*****.**',
message='Change 1 description.',
))
result.add_commit(Commit(
repository_id=result.name, branch='trunk', id=236543,
timestamp=1538050458, order=0,
committer='*****@*****.**',
message='Change 2 description.',
))
result.add_commit(Commit(
repository_id=result.name, branch='trunk', id=236542,
timestamp=1538049108, order=0,
committer='*****@*****.**',
message='Change 3 description.',
))
result.add_commit(Commit(
repository_id=result.name, branch='trunk', id=236541,
timestamp=1538041792, order=0,
committer='*****@*****.**',
message='Change 4 (Part 2) description.',
))
result.add_commit(Commit(
repository_id=result.name, branch='trunk', id=236540,
timestamp=1538029479, order=0,
committer='*****@*****.**',
message='Change 4 (Part 1) description.',
))
result.add_commit(Commit(
repository_id=result.name, branch='safari-606-branch', id=236335,
timestamp=1538029480, order=0,
committer='*****@*****.**',
message='Integration 1.',
))
result.add_commit(Commit(
repository_id=result.name, branch='safari-606-branch', id=236334,
timestamp=1538029479, order=0,
committer='*****@*****.**',
message='Integration 2.',
))
return result
def process_test_results(self,
configuration,
commits,
suite,
test_results,
timestamp=None):
timestamp = timestamp or time.time()
if not self._async_processing:
return self.synchronously_process_test_results(
configuration,
commits,
suite,
test_results=test_results,
timestamp=timestamp)
for branch in self.commit_context.branch_keys_for_commits(commits):
hash_key = hash(configuration) ^ hash(branch) ^ hash(
self.commit_context.uuid_for_commits(commits)) ^ hash(
suite) ^ hash(timestamp)
self.redis.set(
f'{self.QUEUE_NAME}:{hash_key}',
json.dumps(dict(started_processing=0, attempts=0)),
ex=self.PROCESS_TIMEOUT,
)
self.redis.set(
f'data_for_{self.QUEUE_NAME}:{hash_key}',
json.dumps(
dict(
configuration=Configuration.Encoder().default(
configuration),
suite=suite,
commits=Commit.Encoder().default(commits),
timestamp=timestamp,
test_results=test_results,
)),
ex=self.PROCESS_TIMEOUT,
)
return {
key: dict(status='Queued')
for key in list(self._process_upload_callbacks[suite].keys()) +
list(self._process_upload_callbacks[None].keys())
}
def download(self):
AssertRequest.is_type(['GET'])
with self.upload_context:
uploads = self._find_uploads_for_query()
response = []
for config, suite_results in uploads.items():
for suite, results in suite_results.items():
for result in results:
config.sdk = result.get('sdk')
response.append(dict(
configuration=Configuration.Encoder().default(config),
suite=suite,
commits=Commit.Encoder().default(result['commits']),
timestamp=result['timestamp'],
test_results=result['test_results'],
))
return jsonify(response)
def _do_job_for_key(self, key, attempts=1):
job_complete = False
try:
data = json.loads(self.redis.get(f'data_for_{key}'))
self.synchronously_process_test_results(
configuration=Configuration.from_json(data['configuration']),
commits=[Commit.from_json(commit_json) for commit_json in data['commits']],
suite=data['suite'],
timestamp=data['timestamp'],
test_results=data['test_results'],
)
job_complete = True
finally:
if job_complete or attempts >= self.MAX_ATTEMPTS:
self.redis.delete(key)
self.redis.delete(f'data_for_{key}')
else:
self.redis.set(
key,
json.dumps(dict(started_processing=0, attempts=attempts)),
ex=self.PROCESS_TIMEOUT,
)
def find(self):
AssertRequest.is_type()
result = self._find(**request.args.to_dict(flat=False))
if not result:
abort(404, description='No commits found matching the specified criteria')
return jsonify(Commit.Encoder().default(result))