def create(cls, redis, cassandra, async_processing=False): oldest_commit = time.time() for repo in [MockStashRepository.safari(), MockSVNRepository.webkit()]: for commits in repo.commits.values(): for commit in commits: oldest_commit = min( oldest_commit, calendar.timegm(commit.timestamp.timetuple())) model = Model( redis=redis, cassandra=cassandra, repositories=[ MockStashRepository.safari(redis=redis), MockSVNRepository.webkit(redis=redis), ], default_ttl_seconds=time.time() - oldest_commit + Model.TTL_WEEK, async_processing=async_processing, ) with model.commit_context, model.commit_context.cassandra.batch_query_context( ): for repository in model.commit_context.repositories.values(): for branch_commits in repository.commits.values(): for commit in branch_commits: model.commit_context.register_commit(commit) return model
def test_find_range_id(self, client, **kwargs): self.register_all_commits(client) response = client.get( self.URL + '/api/commits?after_id=336610a8&before_id=236540') self.assertEqual(200, response.status_code) self.assertEqual( [Commit.from_json(element) for element in response.json()], [ MockStashRepository.safari().commit_for_id(id='336610a8'), MockStashRepository.safari().commit_for_id(id='bb6bda5f'), MockSVNRepository.webkit().commit_for_id(id=236540), ])
def test_find_range_uuid(self, client, **kwargs): self.register_all_commits(client) response = client.get( self.URL + '/api/commits?after_uuid=153755068501&before_uuid=153756638602') self.assertEqual(200, response.status_code) self.assertEqual( [Commit.from_json(element) for element in response.json()], [ MockStashRepository.safari().commit_for_id(id='7be40842'), MockStashRepository.safari().commit_for_id(id='336610a4') ], )
def test_find_timestamp(self, client, **kwargs): self.register_all_commits(client) response = client.get(self.URL + '/api/commits?timestamp=1537550685') self.assertEqual(200, response.status_code) self.assertEqual(2, len(response.json())) self.assertEqual( [Commit.from_json(element) for element in response.json()], [ MockStashRepository.safari().commit_for_id(id='e64810a4'), MockStashRepository.safari().commit_for_id(id='7be40842'), ]) response = client.get(self.URL + '/api/commits?timestamp=1538041791.8') self.assertEqual(200, response.status_code) self.assertEqual(1, len(response.json())) self.assertEqual(Commit.from_json(response.json()[0]), MockSVNRepository.webkit().commit_for_id(id=236541))
def test_register_with_partial_commit(self, client, **kwargs): self.assertEqual( 200, client.post(self.URL + '/api/commits', data=dict(repository_id='safari', id='bb6bda5f44dd2')).status_code) response = client.get(self.URL + '/api/commits?repository_id=safari') self.assertEqual(200, response.status_code) self.assertEqual( Commit.from_json(response.json()[0]), MockStashRepository.safari().commit_for_id('bb6bda5f44dd2'), ) self.assertEqual( 404, client.post(self.URL + '/api/commits', data=dict(repository_id='safari', id='aaaaaaaaaaaaa')).status_code) self.assertEqual( 200, client.post(self.URL + '/api/commits', data=dict(repository_id='webkit', id='236544')).status_code) response = client.get(self.URL + '/api/commits?repository_id=webkit') self.assertEqual(200, response.status_code) self.assertEqual( Commit.from_json(response.json()[0]), MockSVNRepository.webkit().commit_for_id('236544'), ) self.assertEqual( 404, client.post(self.URL + '/api/commits', data=dict(repository_id='webkit', id='0')).status_code)
def test_branch_stash(self, redis=StrictRedis): git_repo = MockStashRepository.safari(redis=redis()) with self.assertRaises(SCMException): git_repo.commit_for_id('d85222d9407fd', branch='master') commit = git_repo.commit_for_id('d85222d9407fd', branch='safari-606-branch') self.assertEqual(commit.uuid, 153756338300)
def test_upload(self, client, **kwargs): upload_dict = dict( suite='layout-tests', commits=[ MockStashRepository.safari().commit_for_id('bb6bda5f'), MockSVNRepository.webkit().commit_for_id(236542) ], configuration=Configuration.Encoder().default( Configuration( platform='Mac', version='10.14.0', sdk='18A391', is_simulator=False, architecture='x86_64', style='Release', flavor='wk2', )), test_results=MockModelFactory.layout_test_results(), timestamp=int(time.time()), ) response = client.post(self.URL + '/api/upload', data=str( json.dumps(upload_dict, cls=Commit.Encoder))) self.assertEqual(response.status_code, 200) self.assertEqual(response.json()['status'], 'ok') self.assertEqual(response.json()['processing']['python-tests'], dict(status='ok')) response = client.get(self.URL + '/api/upload') self.assertEqual(response.status_code, 200) self.assertEqual(response.json()[0], Commit.Encoder().default(upload_dict))
def test_invalid_upload(self, client, **kwargs): upload_content = 'bad data'.encode('ascii') upload_meta = dict( configuration=json.dumps( ConfigurationContextTest.CONFIGURATIONS[0], cls=Configuration.Encoder), suite='layout-tests', commits=json.dumps([ MockStashRepository.safari().commit_for_id('bb6bda5f'), MockSVNRepository.webkit().commit_for_id(236542) ], cls=Commit.Encoder), ) response = self.upload_file(client, f'{self.URL}/{self.ARCHIVE_API_ENDPOINT}', upload_meta, upload_content) self.assertEqual(response.status_code, 400) self.assertEqual(response.json()['description'], 'Archive is not a zipfile') response = client.post(f'{self.URL}/{self.ARCHIVE_API_ENDPOINT}', data=str(json.dumps(upload_meta))) self.assertEqual(response.status_code, 400) self.assertEqual(response.json()['description'], 'No archive provided')
def register_all_commits(self, client): for repo in [MockStashRepository.safari(), MockSVNRepository.webkit()]: for commits in repo.commits.values(): for commit in commits: self.assertEqual( 200, client.post( self.URL + '/api/commits/register', data=Commit.Encoder().default(commit)).status_code)
def test_colliding_timestamps_stash(self, redis=StrictRedis): git_repo = MockStashRepository.safari(redis=redis()) commit1 = git_repo.commit_for_id('e64810a40', branch='master') commit2 = git_repo.commit_for_id('7be408425', branch='master') self.assertEqual(commit1.timestamp, commit2.timestamp) self.assertNotEqual(commit1.uuid, commit2.uuid) self.assertTrue(commit1 < commit2) self.assertEqual(commit1.order, 0) self.assertEqual(commit2.order, 1)
def test_commit_url(self, redis=StrictRedis, cassandra=CassandraContext): self.init_database(redis=redis, cassandra=cassandra) self.assertEqual( 'https://fake-stash-instance.apple.com/projects/BROWSER/repos/safari/commits/bb6bda5f44dd24d0b54539b8ff6e8c17f519249a', self.database.url( MockStashRepository.safari().commit_for_id(id='bb6bda5f')), ) self.assertEqual( 'https://trac.webkit.org/changeset/236544/webkit', self.database.url( MockSVNRepository.webkit().commit_for_id(id=236544)), )
def test_stash_already_cached(self, redis=StrictRedis): redis_instance = redis() git_repo = MockStashRepository.safari(redis=redis_instance) redis_instance.set( 'repository:' + git_repo.url + '/commits?since=336610a40c3fecb728871e12ca31482ca715b383&until=refs%2Fheads%2Fmaster&limit=1', json.dumps(dict(values=[], size=0, isLastPage=True)), ) self.assertEqual('safari', git_repo.key) commit = git_repo.commit_for_id('336610a4', branch='master') self.assertEqual(commit.uuid, 153756638600)
def init_database(self, redis=StrictRedis, cassandra=CassandraContext): redis_instance = redis() self.stash_repository = MockStashRepository.safari(redis_instance) self.svn_repository = MockSVNRepository.webkit(redis_instance) cassandra.drop_keyspace(keyspace=self.KEYSPACE) self.database = CommitContext( redis=redis_instance, cassandra=cassandra(keyspace=self.KEYSPACE, create_keyspace=True), ) self.database.register_repository(self.stash_repository) self.database.register_repository(self.svn_repository)
def setup_webserver(cls, app, redis=StrictRedis, cassandra=CassandraContext): cassandra.drop_keyspace(keyspace=cls.KEYSPACE) redis_instance = redis() model = Model( redis=redis_instance, cassandra=cassandra(keyspace=cls.KEYSPACE, create_keyspace=True), repositories=[MockSVNRepository.webkit(redis=redis_instance), MockStashRepository.safari(redis=redis_instance)], ) api_routes = APIRoutes(model=model, import_name=__name__) view_routes = ViewRoutes(model=model, controller=api_routes, import_name=__name__) app.register_blueprint(api_routes) app.register_blueprint(view_routes)
def test_find_uuid(self, client, **kwargs): self.register_all_commits(client) response = client.get(self.URL + '/api/commits?uuid=153755068501') self.assertEqual(200, response.status_code) self.assertEqual(1, len(response.json())) self.assertEqual( Commit.from_json(response.json()[0]), MockStashRepository.safari().commit_for_id(id='7be40842')) response = client.get(self.URL + '/api/commits?uuid=153804179200') self.assertEqual(200, response.status_code) self.assertEqual(1, len(response.json())) self.assertEqual(Commit.from_json(response.json()[0]), MockSVNRepository.webkit().commit_for_id(id=236541))
def test_find_id(self, client, **kwargs): self.register_all_commits(client) response = client.get(self.URL + '/api/commits?id=336610a8') self.assertEqual(200, response.status_code) self.assertEqual(1, len(response.json())) self.assertEqual( Commit.from_json(response.json()[0]), MockStashRepository.safari().commit_for_id(id='336610a8')) response = client.get(self.URL + '/api/commits?id=236540') self.assertEqual(200, response.status_code) self.assertEqual(1, len(response.json())) self.assertEqual(Commit.from_json(response.json()[0]), MockSVNRepository.webkit().commit_for_id(id=236540))
def test_previous(self, client, **kwargs): self.register_all_commits(client) response = client.get(self.URL + '/api/commits/previous?id=336610a4') self.assertEqual(200, response.status_code) self.assertEqual(1, len(response.json())) self.assertEqual( MockStashRepository.safari().commit_for_id(id='7be40842'), Commit.from_json(response.json()[0])) response = client.get(self.URL + '/api/commits/previous?id=236542') self.assertEqual(200, response.status_code) self.assertEqual(1, len(response.json())) self.assertEqual(MockSVNRepository.webkit().commit_for_id(id=236541), Commit.from_json(response.json()[0]))
def test_branch_keys_for_commits(self, redis=StrictRedis, cassandra=CassandraContext): self.init_database(redis=redis, cassandra=cassandra) branches = self.database.branch_keys_for_commits([ MockStashRepository.safari().commit_for_id(id='bb6bda5f'), MockSVNRepository.webkit().commit_for_id(id=236544), ]) self.assertEqual(branches, ['default']) branches = self.database.branch_keys_for_commits([ MockStashRepository.safari().commit_for_id( id='79256c32', branch='safari-606-branch'), MockSVNRepository.webkit().commit_for_id(id=236544), ]) self.assertEqual(branches, ['safari-606-branch']) branches = self.database.branch_keys_for_commits([ MockStashRepository.safari().commit_for_id( id='79256c32', branch='safari-606-branch'), MockSVNRepository.webkit().commit_for_id( id=236335, branch='safari-606-branch'), ]) self.assertEqual(branches, ['safari-606-branch'])
def register_archive(self, client): response = client.post( self.URL + '/api/upload/archive', data=dict( configuration=json.dumps( ConfigurationContextTest.CONFIGURATIONS[0], cls=Configuration.Encoder), suite='layout-tests', commits=json.dumps([ MockStashRepository.safari().commit_for_id('bb6bda5f'), MockSVNRepository.webkit().commit_for_id(236542) ], cls=Commit.Encoder), ), files=dict(file=base64.b64decode(MockModelFactory.ARCHIVE_ZIP)), ) self.assertEqual(response.status_code, 200)
def setup_webserver(cls, app, redis=StrictRedis, cassandra=CassandraContext): redis_instance = redis() safari = MockStashRepository.safari(redis_instance) webkit = MockSVNRepository.webkit(redis_instance) cassandra.drop_keyspace(keyspace=cls.KEYSPACE) cassandra_instance = cassandra(keyspace=cls.KEYSPACE, create_keyspace=True) app.register_blueprint( APIRoutes( Model(redis=redis_instance, cassandra=cassandra_instance, repositories=[safari, webkit])))
def test_register_with_full_commit(self, client, **kwargs): git_commit = MockStashRepository.safari().commit_for_id( 'bb6bda5f44dd2') self.assertEqual( 200, client.post(self.URL + '/api/commits', data=Commit.Encoder().default(git_commit)).status_code) response = client.get(self.URL + '/api/commits?repository_id=safari') self.assertEqual(200, response.status_code) self.assertEqual(Commit.from_json(response.json()[0]), git_commit) svn_commit = MockSVNRepository.webkit().commit_for_id('236544') self.assertEqual( 200, client.post(self.URL + '/api/commits', data=Commit.Encoder().default(svn_commit)).status_code) response = client.get(self.URL + '/api/commits?repository_id=webkit') self.assertEqual(200, response.status_code) self.assertEqual(Commit.from_json(response.json()[0]), svn_commit)
def test_upload_and_download(self, client, **kwargs): upload_content = base64.b64decode(MockModelFactory.ARCHIVE_ZIP) upload_meta = dict( configuration=json.dumps( ConfigurationContextTest.CONFIGURATIONS[0], cls=Configuration.Encoder), suite='layout-tests', commits=json.dumps([ MockStashRepository.safari().commit_for_id('bb6bda5f'), MockSVNRepository.webkit().commit_for_id(236542) ], cls=Commit.Encoder), ) response = self.upload_file(client, f'{self.URL}/{self.ARCHIVE_API_ENDPOINT}', upload_meta, upload_content) self.assertEqual(response.status_code, 200) response = client.get(f'{self.URL}/{self.ARCHIVE_API_ENDPOINT}') self.assertEqual(response.status_code, 200) self.assertEqual(response.content, base64.b64decode(MockModelFactory.ARCHIVE_ZIP))
def test_siblings(self, client, **kwargs): self.register_all_commits(client) response = client.get( self.URL + '/api/commits/siblings?repository_id=webkit&id=236542') self.assertEqual(200, response.status_code) commits = { key: [Commit.from_json(element) for element in values] for key, values in response.json().items() } self.assertEqual( commits, { 'safari': [ MockStashRepository.safari().commit_for_id( id='bb6bda5f44dd24') ] }) response = client.get( self.URL + '/api/commits/siblings?repository_id=safari&id=bb6bda5f44dd24') self.assertEqual(200, response.status_code) commits = { key: [Commit.from_json(element) for element in values] for key, values in response.json().items() } self.assertEqual( commits, { 'webkit': [ MockSVNRepository.webkit().commit_for_id(id=236544), MockSVNRepository.webkit().commit_for_id(id=236543), MockSVNRepository.webkit().commit_for_id(id=236542), MockSVNRepository.webkit().commit_for_id(id=236541), MockSVNRepository.webkit().commit_for_id(id=236540), ] })
def test_uuid_for_commits(self): uuid = CommitContext.uuid_for_commits([ MockStashRepository.safari().commit_for_id(id='bb6bda5f'), MockSVNRepository.webkit().commit_for_id(id=236544) ]) self.assertEqual(uuid, 153805240800)
def test_stash(self, redis=StrictRedis): git_repo = MockStashRepository.safari(redis=redis()) self.assertEqual('safari', git_repo.key) commit = git_repo.commit_for_id('bb6bda5f44dd24d', branch='master') self.assertEqual(commit.uuid, 153781028100)
def test_invalid_metadata(self, client, **kwargs): upload_content = base64.b64decode(MockModelFactory.ARCHIVE_ZIP) response = self.upload_file( client, f'{self.URL}/{self.ARCHIVE_API_ENDPOINT}', dict( configuration=json.dumps( ConfigurationContextTest.CONFIGURATIONS[0], cls=Configuration.Encoder), commits=json.dumps([ MockStashRepository.safari().commit_for_id('bb6bda5f'), MockSVNRepository.webkit().commit_for_id(236542) ], cls=Commit.Encoder), ), upload_content) self.assertEqual(response.status_code, 400) self.assertEqual(response.json()['description'], 'No test suite specified') response = self.upload_file( client, f'{self.URL}/{self.ARCHIVE_API_ENDPOINT}', dict( suite='layout-tests', commits=json.dumps([ MockStashRepository.safari().commit_for_id('bb6bda5f'), MockSVNRepository.webkit().commit_for_id(236542) ], cls=Commit.Encoder), ), upload_content) self.assertEqual(response.status_code, 400) self.assertEqual(response.json()['description'], 'Cannot register a partial configuration') response = self.upload_file( client, f'{self.URL}/{self.ARCHIVE_API_ENDPOINT}', dict( configuration=json.dumps( ConfigurationContextTest.CONFIGURATIONS[0], cls=Configuration.Encoder), suite='layout-tests', ), upload_content) self.assertEqual(response.status_code, 400) self.assertEqual(response.json()['description'], 'No commits provided') response = self.upload_file( client, f'{self.URL}/{self.ARCHIVE_API_ENDPOINT}', dict( configuration='Invalid meta-data', suite='layout-tests', commits=json.dumps([ MockStashRepository.safari().commit_for_id('bb6bda5f'), MockSVNRepository.webkit().commit_for_id(236542) ], cls=Commit.Encoder), ), upload_content) self.assertEqual(response.status_code, 400) self.assertEqual( response.json()['description'], 'Invalid configuration, error: Expecting value: line 1 column 1 (char 0)' ) response = self.upload_file( client, f'{self.URL}/{self.ARCHIVE_API_ENDPOINT}', dict( configuration=json.dumps( ConfigurationContextTest.CONFIGURATIONS[0], cls=Configuration.Encoder), suite='layout-tests', commits='Invalid meta-data', ), upload_content) self.assertEqual(response.status_code, 400) self.assertEqual(response.json()['description'], 'Expected commit meta-data to be json')