def create(cls, redis, cassandra, async_processing=False):
oldest_commit = time.time()
for repo in [MockStashRepository.safari(), MockSVNRepository.webkit()]:
for commits in repo.commits.values():
for commit in commits:
oldest_commit = min(
oldest_commit,
calendar.timegm(commit.timestamp.timetuple()))
model = Model(
redis=redis,
cassandra=cassandra,
repositories=[
MockStashRepository.safari(redis=redis),
MockSVNRepository.webkit(redis=redis),
],
default_ttl_seconds=time.time() - oldest_commit + Model.TTL_WEEK,
async_processing=async_processing,
)
with model.commit_context, model.commit_context.cassandra.batch_query_context(
):
for repository in model.commit_context.repositories.values():
for branch_commits in repository.commits.values():
for commit in branch_commits:
model.commit_context.register_commit(commit)
return model
def test_find_range_id(self, client, **kwargs):
self.register_all_commits(client)
response = client.get(
self.URL + '/api/commits?after_id=336610a8&before_id=236540')
self.assertEqual(200, response.status_code)
self.assertEqual(
[Commit.from_json(element) for element in response.json()], [
MockStashRepository.safari().commit_for_id(id='336610a8'),
MockStashRepository.safari().commit_for_id(id='bb6bda5f'),
MockSVNRepository.webkit().commit_for_id(id=236540),
])
def test_find_range_uuid(self, client, **kwargs):
self.register_all_commits(client)
response = client.get(
self.URL +
'/api/commits?after_uuid=153755068501&before_uuid=153756638602')
self.assertEqual(200, response.status_code)
self.assertEqual(
[Commit.from_json(element) for element in response.json()],
[
MockStashRepository.safari().commit_for_id(id='7be40842'),
MockStashRepository.safari().commit_for_id(id='336610a4')
],
)
def test_find_timestamp(self, client, **kwargs):
self.register_all_commits(client)
response = client.get(self.URL + '/api/commits?timestamp=1537550685')
self.assertEqual(200, response.status_code)
self.assertEqual(2, len(response.json()))
self.assertEqual(
[Commit.from_json(element) for element in response.json()], [
MockStashRepository.safari().commit_for_id(id='e64810a4'),
MockStashRepository.safari().commit_for_id(id='7be40842'),
])
response = client.get(self.URL + '/api/commits?timestamp=1538041791.8')
self.assertEqual(200, response.status_code)
self.assertEqual(1, len(response.json()))
self.assertEqual(Commit.from_json(response.json()[0]),
MockSVNRepository.webkit().commit_for_id(id=236541))
def test_register_with_partial_commit(self, client, **kwargs):
self.assertEqual(
200,
client.post(self.URL + '/api/commits',
data=dict(repository_id='safari',
id='bb6bda5f44dd2')).status_code)
response = client.get(self.URL + '/api/commits?repository_id=safari')
self.assertEqual(200, response.status_code)
self.assertEqual(
Commit.from_json(response.json()[0]),
MockStashRepository.safari().commit_for_id('bb6bda5f44dd2'),
)
self.assertEqual(
404,
client.post(self.URL + '/api/commits',
data=dict(repository_id='safari',
id='aaaaaaaaaaaaa')).status_code)
self.assertEqual(
200,
client.post(self.URL + '/api/commits',
data=dict(repository_id='webkit',
id='236544')).status_code)
response = client.get(self.URL + '/api/commits?repository_id=webkit')
self.assertEqual(200, response.status_code)
self.assertEqual(
Commit.from_json(response.json()[0]),
MockSVNRepository.webkit().commit_for_id('236544'),
)
self.assertEqual(
404,
client.post(self.URL + '/api/commits',
data=dict(repository_id='webkit', id='0')).status_code)
def test_branch_stash(self, redis=StrictRedis):
git_repo = MockStashRepository.safari(redis=redis())
with self.assertRaises(SCMException):
git_repo.commit_for_id('d85222d9407fd', branch='master')
commit = git_repo.commit_for_id('d85222d9407fd',
branch='safari-606-branch')
self.assertEqual(commit.uuid, 153756338300)
def test_upload(self, client, **kwargs):
upload_dict = dict(
suite='layout-tests',
commits=[
MockStashRepository.safari().commit_for_id('bb6bda5f'),
MockSVNRepository.webkit().commit_for_id(236542)
],
configuration=Configuration.Encoder().default(
Configuration(
platform='Mac',
version='10.14.0',
sdk='18A391',
is_simulator=False,
architecture='x86_64',
style='Release',
flavor='wk2',
)),
test_results=MockModelFactory.layout_test_results(),
timestamp=int(time.time()),
)
response = client.post(self.URL + '/api/upload',
data=str(
json.dumps(upload_dict,
cls=Commit.Encoder)))
self.assertEqual(response.status_code, 200)
self.assertEqual(response.json()['status'], 'ok')
self.assertEqual(response.json()['processing']['python-tests'],
dict(status='ok'))
response = client.get(self.URL + '/api/upload')
self.assertEqual(response.status_code, 200)
self.assertEqual(response.json()[0],
Commit.Encoder().default(upload_dict))
def test_invalid_upload(self, client, **kwargs):
upload_content = 'bad data'.encode('ascii')
upload_meta = dict(
configuration=json.dumps(
ConfigurationContextTest.CONFIGURATIONS[0],
cls=Configuration.Encoder),
suite='layout-tests',
commits=json.dumps([
MockStashRepository.safari().commit_for_id('bb6bda5f'),
MockSVNRepository.webkit().commit_for_id(236542)
],
cls=Commit.Encoder),
)
response = self.upload_file(client,
f'{self.URL}/{self.ARCHIVE_API_ENDPOINT}',
upload_meta, upload_content)
self.assertEqual(response.status_code, 400)
self.assertEqual(response.json()['description'],
'Archive is not a zipfile')
response = client.post(f'{self.URL}/{self.ARCHIVE_API_ENDPOINT}',
data=str(json.dumps(upload_meta)))
self.assertEqual(response.status_code, 400)
self.assertEqual(response.json()['description'], 'No archive provided')
def register_all_commits(self, client):
for repo in [MockStashRepository.safari(), MockSVNRepository.webkit()]:
for commits in repo.commits.values():
for commit in commits:
self.assertEqual(
200,
client.post(
self.URL + '/api/commits/register',
data=Commit.Encoder().default(commit)).status_code)
def test_colliding_timestamps_stash(self, redis=StrictRedis):
git_repo = MockStashRepository.safari(redis=redis())
commit1 = git_repo.commit_for_id('e64810a40', branch='master')
commit2 = git_repo.commit_for_id('7be408425', branch='master')
self.assertEqual(commit1.timestamp, commit2.timestamp)
self.assertNotEqual(commit1.uuid, commit2.uuid)
self.assertTrue(commit1 < commit2)
self.assertEqual(commit1.order, 0)
self.assertEqual(commit2.order, 1)
def test_commit_url(self, redis=StrictRedis, cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
self.assertEqual(
'https://fake-stash-instance.apple.com/projects/BROWSER/repos/safari/commits/bb6bda5f44dd24d0b54539b8ff6e8c17f519249a',
self.database.url(
MockStashRepository.safari().commit_for_id(id='bb6bda5f')),
)
self.assertEqual(
'https://trac.webkit.org/changeset/236544/webkit',
self.database.url(
MockSVNRepository.webkit().commit_for_id(id=236544)),
)
def test_stash_already_cached(self, redis=StrictRedis):
redis_instance = redis()
git_repo = MockStashRepository.safari(redis=redis_instance)
redis_instance.set(
'repository:' + git_repo.url +
'/commits?since=336610a40c3fecb728871e12ca31482ca715b383&until=refs%2Fheads%2Fmaster&limit=1',
json.dumps(dict(values=[], size=0, isLastPage=True)),
)
self.assertEqual('safari', git_repo.key)
commit = git_repo.commit_for_id('336610a4', branch='master')
self.assertEqual(commit.uuid, 153756638600)
def init_database(self, redis=StrictRedis, cassandra=CassandraContext):
redis_instance = redis()
self.stash_repository = MockStashRepository.safari(redis_instance)
self.svn_repository = MockSVNRepository.webkit(redis_instance)
cassandra.drop_keyspace(keyspace=self.KEYSPACE)
self.database = CommitContext(
redis=redis_instance,
cassandra=cassandra(keyspace=self.KEYSPACE, create_keyspace=True),
)
self.database.register_repository(self.stash_repository)
self.database.register_repository(self.svn_repository)
def setup_webserver(cls, app, redis=StrictRedis, cassandra=CassandraContext):
cassandra.drop_keyspace(keyspace=cls.KEYSPACE)
redis_instance = redis()
model = Model(
redis=redis_instance, cassandra=cassandra(keyspace=cls.KEYSPACE, create_keyspace=True),
repositories=[MockSVNRepository.webkit(redis=redis_instance), MockStashRepository.safari(redis=redis_instance)],
)
api_routes = APIRoutes(model=model, import_name=__name__)
view_routes = ViewRoutes(model=model, controller=api_routes, import_name=__name__)
app.register_blueprint(api_routes)
app.register_blueprint(view_routes)
def test_find_uuid(self, client, **kwargs):
self.register_all_commits(client)
response = client.get(self.URL + '/api/commits?uuid=153755068501')
self.assertEqual(200, response.status_code)
self.assertEqual(1, len(response.json()))
self.assertEqual(
Commit.from_json(response.json()[0]),
MockStashRepository.safari().commit_for_id(id='7be40842'))
response = client.get(self.URL + '/api/commits?uuid=153804179200')
self.assertEqual(200, response.status_code)
self.assertEqual(1, len(response.json()))
self.assertEqual(Commit.from_json(response.json()[0]),
MockSVNRepository.webkit().commit_for_id(id=236541))
def test_find_id(self, client, **kwargs):
self.register_all_commits(client)
response = client.get(self.URL + '/api/commits?id=336610a8')
self.assertEqual(200, response.status_code)
self.assertEqual(1, len(response.json()))
self.assertEqual(
Commit.from_json(response.json()[0]),
MockStashRepository.safari().commit_for_id(id='336610a8'))
response = client.get(self.URL + '/api/commits?id=236540')
self.assertEqual(200, response.status_code)
self.assertEqual(1, len(response.json()))
self.assertEqual(Commit.from_json(response.json()[0]),
MockSVNRepository.webkit().commit_for_id(id=236540))
def test_previous(self, client, **kwargs):
self.register_all_commits(client)
response = client.get(self.URL + '/api/commits/previous?id=336610a4')
self.assertEqual(200, response.status_code)
self.assertEqual(1, len(response.json()))
self.assertEqual(
MockStashRepository.safari().commit_for_id(id='7be40842'),
Commit.from_json(response.json()[0]))
response = client.get(self.URL + '/api/commits/previous?id=236542')
self.assertEqual(200, response.status_code)
self.assertEqual(1, len(response.json()))
self.assertEqual(MockSVNRepository.webkit().commit_for_id(id=236541),
Commit.from_json(response.json()[0]))
def test_branch_keys_for_commits(self,
redis=StrictRedis,
cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
branches = self.database.branch_keys_for_commits([
MockStashRepository.safari().commit_for_id(id='bb6bda5f'),
MockSVNRepository.webkit().commit_for_id(id=236544),
])
self.assertEqual(branches, ['default'])
branches = self.database.branch_keys_for_commits([
MockStashRepository.safari().commit_for_id(
id='79256c32', branch='safari-606-branch'),
MockSVNRepository.webkit().commit_for_id(id=236544),
])
self.assertEqual(branches, ['safari-606-branch'])
branches = self.database.branch_keys_for_commits([
MockStashRepository.safari().commit_for_id(
id='79256c32', branch='safari-606-branch'),
MockSVNRepository.webkit().commit_for_id(
id=236335, branch='safari-606-branch'),
])
self.assertEqual(branches, ['safari-606-branch'])
def register_archive(self, client):
response = client.post(
self.URL + '/api/upload/archive',
data=dict(
configuration=json.dumps(
ConfigurationContextTest.CONFIGURATIONS[0],
cls=Configuration.Encoder),
suite='layout-tests',
commits=json.dumps([
MockStashRepository.safari().commit_for_id('bb6bda5f'),
MockSVNRepository.webkit().commit_for_id(236542)
],
cls=Commit.Encoder),
),
files=dict(file=base64.b64decode(MockModelFactory.ARCHIVE_ZIP)),
)
self.assertEqual(response.status_code, 200)
def setup_webserver(cls,
app,
redis=StrictRedis,
cassandra=CassandraContext):
redis_instance = redis()
safari = MockStashRepository.safari(redis_instance)
webkit = MockSVNRepository.webkit(redis_instance)
cassandra.drop_keyspace(keyspace=cls.KEYSPACE)
cassandra_instance = cassandra(keyspace=cls.KEYSPACE,
create_keyspace=True)
app.register_blueprint(
APIRoutes(
Model(redis=redis_instance,
cassandra=cassandra_instance,
repositories=[safari, webkit])))
def test_register_with_full_commit(self, client, **kwargs):
git_commit = MockStashRepository.safari().commit_for_id(
'bb6bda5f44dd2')
self.assertEqual(
200,
client.post(self.URL + '/api/commits',
data=Commit.Encoder().default(git_commit)).status_code)
response = client.get(self.URL + '/api/commits?repository_id=safari')
self.assertEqual(200, response.status_code)
self.assertEqual(Commit.from_json(response.json()[0]), git_commit)
svn_commit = MockSVNRepository.webkit().commit_for_id('236544')
self.assertEqual(
200,
client.post(self.URL + '/api/commits',
data=Commit.Encoder().default(svn_commit)).status_code)
response = client.get(self.URL + '/api/commits?repository_id=webkit')
self.assertEqual(200, response.status_code)
self.assertEqual(Commit.from_json(response.json()[0]), svn_commit)
def test_upload_and_download(self, client, **kwargs):
upload_content = base64.b64decode(MockModelFactory.ARCHIVE_ZIP)
upload_meta = dict(
configuration=json.dumps(
ConfigurationContextTest.CONFIGURATIONS[0],
cls=Configuration.Encoder),
suite='layout-tests',
commits=json.dumps([
MockStashRepository.safari().commit_for_id('bb6bda5f'),
MockSVNRepository.webkit().commit_for_id(236542)
],
cls=Commit.Encoder),
)
response = self.upload_file(client,
f'{self.URL}/{self.ARCHIVE_API_ENDPOINT}',
upload_meta, upload_content)
self.assertEqual(response.status_code, 200)
response = client.get(f'{self.URL}/{self.ARCHIVE_API_ENDPOINT}')
self.assertEqual(response.status_code, 200)
self.assertEqual(response.content,
base64.b64decode(MockModelFactory.ARCHIVE_ZIP))
def test_siblings(self, client, **kwargs):
self.register_all_commits(client)
response = client.get(
self.URL + '/api/commits/siblings?repository_id=webkit&id=236542')
self.assertEqual(200, response.status_code)
commits = {
key: [Commit.from_json(element) for element in values]
for key, values in response.json().items()
}
self.assertEqual(
commits, {
'safari': [
MockStashRepository.safari().commit_for_id(
id='bb6bda5f44dd24')
]
})
response = client.get(
self.URL +
'/api/commits/siblings?repository_id=safari&id=bb6bda5f44dd24')
self.assertEqual(200, response.status_code)
commits = {
key: [Commit.from_json(element) for element in values]
for key, values in response.json().items()
}
self.assertEqual(
commits, {
'webkit': [
MockSVNRepository.webkit().commit_for_id(id=236544),
MockSVNRepository.webkit().commit_for_id(id=236543),
MockSVNRepository.webkit().commit_for_id(id=236542),
MockSVNRepository.webkit().commit_for_id(id=236541),
MockSVNRepository.webkit().commit_for_id(id=236540),
]
})
def test_uuid_for_commits(self):
uuid = CommitContext.uuid_for_commits([
MockStashRepository.safari().commit_for_id(id='bb6bda5f'),
MockSVNRepository.webkit().commit_for_id(id=236544)
])
self.assertEqual(uuid, 153805240800)
def test_stash(self, redis=StrictRedis):
git_repo = MockStashRepository.safari(redis=redis())
self.assertEqual('safari', git_repo.key)
commit = git_repo.commit_for_id('bb6bda5f44dd24d', branch='master')
self.assertEqual(commit.uuid, 153781028100)
def test_invalid_metadata(self, client, **kwargs):
upload_content = base64.b64decode(MockModelFactory.ARCHIVE_ZIP)
response = self.upload_file(
client, f'{self.URL}/{self.ARCHIVE_API_ENDPOINT}',
dict(
configuration=json.dumps(
ConfigurationContextTest.CONFIGURATIONS[0],
cls=Configuration.Encoder),
commits=json.dumps([
MockStashRepository.safari().commit_for_id('bb6bda5f'),
MockSVNRepository.webkit().commit_for_id(236542)
],
cls=Commit.Encoder),
), upload_content)
self.assertEqual(response.status_code, 400)
self.assertEqual(response.json()['description'],
'No test suite specified')
response = self.upload_file(
client, f'{self.URL}/{self.ARCHIVE_API_ENDPOINT}',
dict(
suite='layout-tests',
commits=json.dumps([
MockStashRepository.safari().commit_for_id('bb6bda5f'),
MockSVNRepository.webkit().commit_for_id(236542)
],
cls=Commit.Encoder),
), upload_content)
self.assertEqual(response.status_code, 400)
self.assertEqual(response.json()['description'],
'Cannot register a partial configuration')
response = self.upload_file(
client, f'{self.URL}/{self.ARCHIVE_API_ENDPOINT}',
dict(
configuration=json.dumps(
ConfigurationContextTest.CONFIGURATIONS[0],
cls=Configuration.Encoder),
suite='layout-tests',
), upload_content)
self.assertEqual(response.status_code, 400)
self.assertEqual(response.json()['description'], 'No commits provided')
response = self.upload_file(
client, f'{self.URL}/{self.ARCHIVE_API_ENDPOINT}',
dict(
configuration='Invalid meta-data',
suite='layout-tests',
commits=json.dumps([
MockStashRepository.safari().commit_for_id('bb6bda5f'),
MockSVNRepository.webkit().commit_for_id(236542)
],
cls=Commit.Encoder),
), upload_content)
self.assertEqual(response.status_code, 400)
self.assertEqual(
response.json()['description'],
'Invalid configuration, error: Expecting value: line 1 column 1 (char 0)'
)
response = self.upload_file(
client, f'{self.URL}/{self.ARCHIVE_API_ENDPOINT}',
dict(
configuration=json.dumps(
ConfigurationContextTest.CONFIGURATIONS[0],
cls=Configuration.Encoder),
suite='layout-tests',
commits='Invalid meta-data',
), upload_content)
self.assertEqual(response.status_code, 400)
self.assertEqual(response.json()['description'],
'Expected commit meta-data to be json')
