def register_all_commits(self, client): for repo in [MockStashRepository.safari(), MockSVNRepository.webkit()]: for commits in repo.commits.values(): for commit in commits: self.assertEqual( 200, client.post( self.URL + '/api/commits/register', data=Commit.Encoder().default(commit)).status_code)
def test_udid(self): self.assertEqual( Commit( repository_id='safari', branch='master', id='7be4084258a452e8fe22f36287c5b321e9c8249b', timestamp=1537550685, order=1, ).uuid, 153755068501)
def to_commit(self): return Commit( repository_id=self.repository_id, branch=self.branch, id=self.commit_id, timestamp=self.uuid // Commit.TIMESTAMP_TO_UUID_MULTIPLIER, order=self.uuid % Commit.TIMESTAMP_TO_UUID_MULTIPLIER, committer=self.committer, message=self.message, )
def test_register_with_full_commit(self, client, **kwargs): git_commit = MockStashRepository.safari().commit_for_id( 'bb6bda5f44dd2') self.assertEqual( 200, client.post(self.URL + '/api/commits', data=Commit.Encoder().default(git_commit)).status_code) response = client.get(self.URL + '/api/commits?repository_id=safari') self.assertEqual(200, response.status_code) self.assertEqual(Commit.from_json(response.json()[0]), git_commit) svn_commit = MockSVNRepository.webkit().commit_for_id('236544') self.assertEqual( 200, client.post(self.URL + '/api/commits', data=Commit.Encoder().default(svn_commit)).status_code) response = client.get(self.URL + '/api/commits?repository_id=webkit') self.assertEqual(200, response.status_code) self.assertEqual(Commit.from_json(response.json()[0]), svn_commit)
def test_find_range_id(self, client, **kwargs): self.register_all_commits(client) response = client.get( self.URL + '/api/commits?after_id=336610a8&before_id=236540') self.assertEqual(200, response.status_code) self.assertEqual( [Commit.from_json(element) for element in response.json()], [ MockStashRepository.safari().commit_for_id(id='336610a8'), MockStashRepository.safari().commit_for_id(id='bb6bda5f'), MockSVNRepository.webkit().commit_for_id(id=236540), ])
def test_find_range_id(self, client, **kwargs): self.register_all_commits(client) response = client.get(self.URL + '/api/commits?after_id=6&before_id=fff83bb2d917') self.assertEqual(200, response.status_code) self.assertEqual( [Commit.from_json(element).id for element in response.json()], [ '6', '9b8311f25a77ba14923d9d5a6532103f54abefcb', 'fff83bb2d9171b4d9196e977eb0508fd57e7a08d', ])
def commit_for_id(self, id): commit = self.remote.commit(revision=id, include_identifier=False) return Commit( repository_id=self.key, id=commit.revision, branch=commit.branch, timestamp=commit.timestamp, order=commit.order, committer=commit.author.email, message=commit.message, )
def test_compare(self): commit1 = Commit( repository_id='safari', branch='master', id='e64810a40c3fecb728871e12ca31482ca715b383', timestamp=1537550685, ) commit2 = Commit( repository_id='safari', branch='master', id='7be4084258a452e8fe22f36287c5b321e9c8249b', timestamp=1537550685, order=1, ) commit3 = Commit( repository_id='safari', branch='master', id='bb6bda5f44dd24d0b54539b8ff6e8c17f519249a', timestamp=1537810281, ) commit4 = Commit( repository_id='webkit', branch='master', id=236522, timestamp=1537826614, ) self.assertTrue(commit2 > commit1) self.assertTrue(commit2 >= commit1) self.assertTrue(commit1 >= commit1) self.assertTrue(commit1 < commit2) self.assertTrue(commit1 <= commit2) self.assertTrue(commit1 <= commit1) self.assertTrue(commit1 == commit1) self.assertTrue(commit2 < commit3) self.assertTrue(commit3 > commit2) self.assertEqual(commit1.timestamp, commit2.timestamp) self.assertTrue(commit4 > commit3) self.assertNotEqual(commit3.repository_id, commit4.repository_id)
def unpack(self): return dict( commits=[ Commit.from_json(element) for element in json.loads( UploadContext.from_zip(bytearray(self.commits))) ], sdk=None if self.sdk == '?' else self.sdk, test_results=json.loads( UploadContext.from_zip(bytearray(self.test_results))), timestamp=calendar.timegm(self.time_uploaded.timetuple()), version=self.upload_version, )
def run_function_through_redis_cache(self, key, function): result = self.redis.get('commit_mapping:' + key) if result: try: result = [Commit.from_json(element) for element in json.loads(result)] except ValueError: result = None if result is None: result = function() if result: self.redis.set('commit_mapping:' + key, json.dumps(result, cls=Commit.Encoder), ex=self.cache_timeout) return result
def test_register_with_full_commit(self, client, **kwargs): git_commit = Commit( repository_id='safari', branch='main', id='8eba280e32daaf5fddbbb65aea37932ea9ad85df', timestamp=1601650100, order=0, committer='*****@*****.**', message='custom commit', ) self.assertEqual( 200, client.post(self.URL + '/api/commits', data=Commit.Encoder().default(git_commit)).status_code) response = client.get(self.URL + '/api/commits?repository_id=safari') self.assertEqual(200, response.status_code) self.assertEqual(Commit.from_json(response.json()[0]), git_commit) svn_commit = Commit( repository_id='webkit', branch='trunk', id='1234', timestamp=1601650100, order=0, committer='*****@*****.**', message='custom commit', ) self.assertEqual( 200, client.post(self.URL + '/api/commits', data=Commit.Encoder().default(svn_commit)).status_code) response = client.get(self.URL + '/api/commits?repository_id=webkit') self.assertEqual(200, response.status_code) self.assertEqual(Commit.from_json(response.json()[0]), svn_commit)
def _find_comparison(commit_context, repository_id, branch, id, uuid, timestamp, priority=min): if bool(id) + bool(uuid) + bool(timestamp) > 1: abort( 400, description= 'Can only search by one of [commit id, commit uuid, timestamp] in a single request' ) try: if uuid: # We don't need real commit to search by uuid and CommitContexts have trouble diffrentiating between uuids and timestamps. uuid = priority([int(element) for element in uuid]) return Commit('?', '?', '?', uuid // Commit.TIMESTAMP_TO_UUID_MULTIPLIER, uuid % Commit.TIMESTAMP_TO_UUID_MULTIPLIER) if timestamp: return priority([round(float(element)) for element in timestamp]) except ValueError: abort(400, description='Timestamp and uuid must be integers') if not repository_id and not id: return None if not repository_id: repository_id = commit_context.repositories.keys() for repository in repository_id: if repository not in commit_context.repositories.keys(): abort( 404, description=f"\'{repository}\' is not a registered repository") result = [] for repository in repository_id: for b in branch: if id: for elm in id: result += commit_context.find_commits_by_id( repository, b, elm) else: result += commit_context.find_commits_in_range(repository, b, limit=1) if not result: abort(404, description='No commits found matching the specified criteria') return priority(result)
def test_find_range_uuid(self, client, **kwargs): self.register_all_commits(client) response = client.get( self.URL + '/api/commits?after_uuid=160166800000&before_uuid=160166800001') self.assertEqual(200, response.status_code) self.assertEqual( [Commit.from_json(element).hash for element in response.json()], [ 'bae5d1e90999d4f916a8a15810ccfa43f37a2fd6', 'd8bce26fa65c6fc8f39c17927abb77f69fab82fc', ], )
def test_find_range_uuid(self, client, **kwargs): self.register_all_commits(client) response = client.get( self.URL + '/api/commits?after_uuid=153755068501&before_uuid=153756638602') self.assertEqual(200, response.status_code) self.assertEqual( [Commit.from_json(element) for element in response.json()], [ MockStashRepository.safari().commit_for_id(id='7be40842'), MockStashRepository.safari().commit_for_id(id='336610a4') ], )
def previous(self, limit=None, **kwargs): AssertRequest.is_type() AssertRequest.query_kwargs_empty(limit=limit) with self.commit_context: commits = self._find(**kwargs) if not commits: abort(404, description='No commits found matching the specified criteria') if len(commits) > 1: abort(404, description=f'{len(commits)} commits found matching the specified criteria') commit = self.commit_context.previous_commit(commits[0]) return jsonify(Commit.Encoder().default([commit] if commit else []))
def test_register_with_partial_commit(self, client, **kwargs): with MockModelFactory.safari(), MockModelFactory.webkit( ), OutputCapture(): self.assertEqual( 200, client.post(self.URL + '/api/commits', data=dict(repository_id='safari', id='d8bce26fa65c')).status_code) response = client.get(self.URL + '/api/commits?repository_id=safari') self.assertEqual(200, response.status_code) self.assertEqual( Commit.from_json(response.json()[0]).message, 'Patch Series\n', ) self.assertEqual( 404, client.post(self.URL + '/api/commits', data=dict(repository_id='safari', id='aaaaaaaaaaaaa')).status_code) self.assertEqual( 200, client.post(self.URL + '/api/commits', data=dict(repository_id='webkit', id='6')).status_code) response = client.get(self.URL + '/api/commits?repository_id=webkit') self.assertEqual(200, response.status_code) self.assertEqual( Commit.from_json(response.json()[0]).message, '6th commit', ) self.assertEqual( 404, client.post(self.URL + '/api/commits', data=dict(repository_id='webkit', id='1234')).status_code)
def test_find_range_timestamp(self, client, **kwargs): self.register_all_commits(client) response = client.get( self.URL + '/api/commits?after_timestamp=1538041792.3&before_timestamp=1538049108' ) self.assertEqual(200, response.status_code) self.assertEqual( [Commit.from_json(element) for element in response.json()], [ MockSVNRepository.webkit().commit_for_id(id=236541), MockSVNRepository.webkit().commit_for_id(id=236542) ], )
def siblings(self, limit=None, **kwargs): AssertRequest.is_type() AssertRequest.query_kwargs_empty(limit=limit) with self.commit_context: commits = self._find(**kwargs) if not commits: abort(404, description='No commits found matching the specified criteria') if len(commits) > 1: abort(404, description=f'{len(commits)} commits found matching the specified criteria') repositories = sorted(self.commit_context.repositories.keys()) repositories.remove(commits[0].repository_id) return jsonify(Commit.Encoder().default(self.commit_context.sibling_commits(commits[0], repositories)))
def test_find_range_timestamp(self, client, **kwargs): self.register_all_commits(client) response = client.get( self.URL + '/api/commits?after_timestamp=1601637900&before_timestamp=1601639900' ) self.assertEqual(200, response.status_code) self.assertEqual( [ Commit.from_json(element).revision for element in response.json() ], [4, 6], )
def test_register_via_post(self, client, **kwargs): self.assertEqual( 200, client.post(self.URL + '/api/commits', data=dict(repository_id='safari', id='bb6bda5f44dd2')).status_code) response = client.get(self.URL + '/api/commits?repository_id=safari') self.assertEqual(200, response.status_code) self.assertEqual( Commit.from_json(response.json()[0]), MockStashRepository.safari().commit_for_id('bb6bda5f44dd2'), ) self.assertEqual( 200, client.post(self.URL + '/api/commits', data=dict(repository_id='webkit', id='236544')).status_code) response = client.get(self.URL + '/api/commits?repository_id=webkit') self.assertEqual(200, response.status_code) self.assertEqual( Commit.from_json(response.json()[0]), MockSVNRepository.webkit().commit_for_id('236544'), )
def test_siblings(self, client, **kwargs): self.register_all_commits(client) response = client.get( self.URL + '/api/commits/siblings?repository_id=webkit&id=236542') self.assertEqual(200, response.status_code) commits = { key: [Commit.from_json(element) for element in values] for key, values in response.json().items() } self.assertEqual( commits, { 'safari': [ MockStashRepository.safari().commit_for_id( id='bb6bda5f44dd24') ] }) response = client.get( self.URL + '/api/commits/siblings?repository_id=safari&id=bb6bda5f44dd24') self.assertEqual(200, response.status_code) commits = { key: [Commit.from_json(element) for element in values] for key, values in response.json().items() } self.assertEqual( commits, { 'webkit': [ MockSVNRepository.webkit().commit_for_id(id=236544), MockSVNRepository.webkit().commit_for_id(id=236543), MockSVNRepository.webkit().commit_for_id(id=236542), MockSVNRepository.webkit().commit_for_id(id=236541), MockSVNRepository.webkit().commit_for_id(id=236540), ] })
def commit_for_id(self, id, branch=Repository.DEFAULT_BRANCH): try: with self.session(): commit_data = self.get(f'commits/{id}') if not commit_data: raise SCMException( f'Commit {id} does not exist on branch {branch}') timestamp = int(commit_data['committerTimestamp'] ) // self.COMMIT_TIMESTAMP_CONVERSION branch_filter = urllib.parse.quote(f'refs/heads/{branch}') commits_between_commit_and_branch_head = self.get( f"commits?since={commit_data['id']}&until={branch_filter}&limit=1", cache=False) if commits_between_commit_and_branch_head[ 'isLastPage'] and commits_between_commit_and_branch_head[ 'size'] == 0: # We may have missed the case the commit in question is the HEAD of the branch commits_at_branch_head = self.get( f'commits?until={branch_filter}&limit=20', cache=False) if all([ commit['id'] != commit_data['id'] for commit in commits_at_branch_head['values'] ]): raise SCMException(f'{id} exists, but not on {branch}') # Ordering commits by timestamp in git is a bit problematic because multiple commits can share a timestamp. # Generally, if your parent shares your timestamp, your order needs to be incremented. order = 0 loop_data = commit_data while len(loop_data['parents']) == 1 and int( loop_data['parents'][0]['committerTimestamp'] ) // self.COMMIT_TIMESTAMP_CONVERSION == timestamp: order += 1 loop_data = self.get( f"commits/{loop_data['parents'][0]['id']}") return Commit( repository_id=self.key, id=commit_data['id'], branch=branch, timestamp=timestamp, order=order, committer=commit_data['committer']['emailAddress'], message=commit_data['message'], ) except ValueError: raise SCMException(f'Failed to connect to {self.url}')
def test_basic_svn(self, redis=StrictRedis): svn_repo = MockSVNRepository(url='svn.webkit.org/repository/webkit/', name='webkit', redis=redis()) svn_repo.add_commit( Commit( repository_id=svn_repo.name, branch='trunk', id=236544, timestamp=1538052408, order=0, committer='*****@*****.**', message='Change 1 description.', )) commit = svn_repo.commit_for_id(236544, branch='trunk') self.assertEqual(commit.uuid, 153805240800) self.assertIsNone(commit.message, None)
def commit(self, **kwargs): with self.commit_context: commit = self._single_commit() repositories = list(self.commit_context.repositories.keys()) repositories.remove(commit.repository_id) siblings = self.commit_context.sibling_commits( commit, repositories) repositories = [commit.repository_id] + sorted( [str(key) for key, lst in siblings.items() if lst]) return self.environment.get_template('commit.html').render( title=self.site_menu.title + ': ' + str(commit.id), commit=commit, repository_ids=repositories, commits=Commit.Encoder().default( [commit] + [item for lst in siblings.values() for item in lst]), **kwargs)
def safari(redis=None): result = MockStashRepository('https://fake-stash-instance.apple.com/projects/BROWSER/repos/safari', name='safari', redis=redis) result.add_commit(Commit( repository_id=result.name, branch='master', id='bb6bda5f44dd24d0b54539b8ff6e8c17f519249a', timestamp=1537810281, order=0, committer='*****@*****.**', message='Change 1 description.', )) result.add_commit(Commit( repository_id=result.name, branch='master', id='336610a84fdcf14ddcf1db65075af95480516fda', timestamp=1537809818, order=0, committer='*****@*****.**', message='Change 2 description.', )) result.add_commit(Commit( repository_id=result.name, branch='master', id='336610a40c3fecb728871e12ca31482ca715b383', timestamp=1537566386, order=0, committer='*****@*****.**', message='<rdar://problem/99999999> Change 3 description.', )) result.add_commit(Commit( repository_id=result.name, branch='master', id='e64810a40c3fecb728871e12ca31482ca715b383', timestamp=1537550685, order=0, committer='*****@*****.**', message=u'Change 4 \u2014 (Part 1) description.', )) result.add_commit(Commit( repository_id=result.name, branch='master', id='7be4084258a452e8fe22f36287c5b321e9c8249b', timestamp=1537550685, order=1, committer='*****@*****.**', message=u'Change 4 \u2014 (Part 2) description.\nReviewed by person.', )) result.add_commit(Commit( repository_id=result.name, branch='safari-606-branch', id='79256c32a855ac8612112279008334d90e901c55', timestamp=1537897367, order=0, committer='*****@*****.**', message='Change 5 description.', )) result.add_commit(Commit( repository_id=result.name, branch='safari-606-branch', id='d85222d9407fdbbf47406509400a9cecb73ac6de', timestamp=1537563383, order=0, committer='*****@*****.**', message='Change 6 description.', )) return result
def webkit(redis=None): result = MockWebKitRepository(redis=redis) result.add_commit(Commit( repository_id=result.name, branch='trunk', id=236544, timestamp=1538052408, order=0, committer='*****@*****.**', message='Change 1 description.', )) result.add_commit(Commit( repository_id=result.name, branch='trunk', id=236543, timestamp=1538050458, order=0, committer='*****@*****.**', message='Change 2 description.', )) result.add_commit(Commit( repository_id=result.name, branch='trunk', id=236542, timestamp=1538049108, order=0, committer='*****@*****.**', message='Change 3 description.', )) result.add_commit(Commit( repository_id=result.name, branch='trunk', id=236541, timestamp=1538041792, order=0, committer='*****@*****.**', message='Change 4 (Part 2) description.', )) result.add_commit(Commit( repository_id=result.name, branch='trunk', id=236540, timestamp=1538029479, order=0, committer='*****@*****.**', message='Change 4 (Part 1) description.', )) result.add_commit(Commit( repository_id=result.name, branch='safari-606-branch', id=236335, timestamp=1538029480, order=0, committer='*****@*****.**', message='Integration 1.', )) result.add_commit(Commit( repository_id=result.name, branch='safari-606-branch', id=236334, timestamp=1538029479, order=0, committer='*****@*****.**', message='Integration 2.', )) return result
def process_test_results(self, configuration, commits, suite, test_results, timestamp=None): timestamp = timestamp or time.time() if not self._async_processing: return self.synchronously_process_test_results( configuration, commits, suite, test_results=test_results, timestamp=timestamp) for branch in self.commit_context.branch_keys_for_commits(commits): hash_key = hash(configuration) ^ hash(branch) ^ hash( self.commit_context.uuid_for_commits(commits)) ^ hash( suite) ^ hash(timestamp) self.redis.set( f'{self.QUEUE_NAME}:{hash_key}', json.dumps(dict(started_processing=0, attempts=0)), ex=self.PROCESS_TIMEOUT, ) self.redis.set( f'data_for_{self.QUEUE_NAME}:{hash_key}', json.dumps( dict( configuration=Configuration.Encoder().default( configuration), suite=suite, commits=Commit.Encoder().default(commits), timestamp=timestamp, test_results=test_results, )), ex=self.PROCESS_TIMEOUT, ) return { key: dict(status='Queued') for key in list(self._process_upload_callbacks[suite].keys()) + list(self._process_upload_callbacks[None].keys()) }
def download(self): AssertRequest.is_type(['GET']) with self.upload_context: uploads = self._find_uploads_for_query() response = [] for config, suite_results in uploads.items(): for suite, results in suite_results.items(): for result in results: config.sdk = result.get('sdk') response.append(dict( configuration=Configuration.Encoder().default(config), suite=suite, commits=Commit.Encoder().default(result['commits']), timestamp=result['timestamp'], test_results=result['test_results'], )) return jsonify(response)
def _do_job_for_key(self, key, attempts=1): job_complete = False try: data = json.loads(self.redis.get(f'data_for_{key}')) self.synchronously_process_test_results( configuration=Configuration.from_json(data['configuration']), commits=[Commit.from_json(commit_json) for commit_json in data['commits']], suite=data['suite'], timestamp=data['timestamp'], test_results=data['test_results'], ) job_complete = True finally: if job_complete or attempts >= self.MAX_ATTEMPTS: self.redis.delete(key) self.redis.delete(f'data_for_{key}') else: self.redis.set( key, json.dumps(dict(started_processing=0, attempts=attempts)), ex=self.PROCESS_TIMEOUT, )
def find(self): AssertRequest.is_type() result = self._find(**request.args.to_dict(flat=False)) if not result: abort(404, description='No commits found matching the specified criteria') return jsonify(Commit.Encoder().default(result))