def find_test_results(self,
configurations,
suite,
branch=None,
begin=None,
end=None,
recent=True,
limit=100):
if not isinstance(suite, str):
raise TypeError(f'Expected type {str}, got {type(suite)}')
with self:
result = {}
for configuration in configurations:
result.update({
config: [value.unpack() for value in values]
for config, values in self.configuration_context.
select_from_table_with_configurations(
self.UploadsByConfiguration.__table_name__,
configurations=[configuration],
recent=recent,
suite=suite,
sdk=configuration.sdk,
branch=branch
or self.commit_context.DEFAULT_BRANCH_KEY,
uuid__gte=CommitContext.convert_to_uuid(begin),
uuid__lte=CommitContext.convert_to_uuid(
end, CommitContext.timestamp_to_uuid()),
limit=limit,
).items()
})
return result
def _find_results(
self, table, configurations, suite, recent=True,
branch=None, begin=None, end=None,
begin_query_time=None, end_query_time=None,
limit=DEFAULT_LIMIT,
):
if not isinstance(suite, str):
raise TypeError(f'Expected type {str}, got {type(suite)}')
def get_time(time):
if isinstance(time, datetime):
return time
elif time:
return datetime.utcfromtimestamp(int(time))
return None
with self:
result = {}
for configuration in configurations:
result.update({config: [value.unpack() for value in values] for config, values in self.configuration_context.select_from_table_with_configurations(
table.__table_name__, configurations=[configuration], recent=recent,
suite=suite, sdk=configuration.sdk, branch=branch or self.commit_context.DEFAULT_BRANCH_KEY,
uuid__gte=CommitContext.convert_to_uuid(begin),
uuid__lte=CommitContext.convert_to_uuid(end, CommitContext.timestamp_to_uuid()),
start_time__gte=get_time(begin_query_time), start_time__lte=get_time(end_query_time),
limit=limit,
).items()})
return result
def __init__(self, redis, cassandra, repositories=[], default_ttl_seconds=TTL_YEAR * 5, archive_ttl_seconds=TTL_WEEK * 8, async_processing=False, s3_credentials=None):
if default_ttl_seconds is not None and default_ttl_seconds < 4 * self.TTL_WEEK:
raise ValueError('TTL must be at least 4 weeks')
if archive_ttl_seconds is not None and archive_ttl_seconds < 2 * self.TTL_WEEK:
raise ValueError('Archive TTL must be at least 2 weeks')
self.default_ttl_seconds = default_ttl_seconds
self.archive_ttl_seconds = archive_ttl_seconds or default_ttl_seconds
self._async_processing = async_processing
self.redis = redis
self.cassandra = cassandra
with self.cassandra:
self.cassandra.create_table(self.HealthTable)
self.commit_context = CommitContext(redis, cassandra)
for repository in repositories:
self.commit_context.register_repository(repository)
self.configuration_context = ConfigurationContext(redis, cassandra)
self.upload_context = UploadContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
async_processing=async_processing,
)
self.suite_context = SuiteContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
self.test_context = TestContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
self.failure_context = FailureContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
self.ci_context = CIContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
for context in [self.ci_context, self.failure_context, self.suite_context, self.test_context]:
self.upload_context.register_upload_callback(context.name, context.register)
self.archive_context = ArchiveContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.archive_ttl_seconds,
s3_credentials=s3_credentials,
)
def find_archive(
self, configurations=None, suite=None, recent=True,
branch=None, begin=None, end=None,
begin_query_time=None, end_query_time=None,
limit=DEFAULT_LIMIT,
):
if not configurations:
configurations = []
if not isinstance(suite, str):
raise TypeError(f'Expected type {str}, got {type(suite)}')
with self:
metadata_by_config = {}
for configuration in configurations:
metadata_by_config.update({config: [value.unpack() for value in values] for config, values in self.configuration_context.select_from_table_with_configurations(
self.ArchiveMetaDataByCommit.__table_name__, configurations=[configuration], recent=recent,
suite=suite, sdk=configuration.sdk, branch=branch or self.commit_context.DEFAULT_BRANCH_KEY,
uuid__gte=CommitContext.convert_to_uuid(begin),
uuid__lte=CommitContext.convert_to_uuid(end, CommitContext.timestamp_to_uuid()),
start_time__gte=_get_time(begin_query_time), start_time__lte=_get_time(end_query_time),
limit=limit,
).items()})
memory_used = 0
for values in metadata_by_config.values():
for value in values:
if not value.get('digest'):
continue
memory_used += value.get('size', 0)
if memory_used > self.MEMORY_LIMIT:
raise RuntimeError('Hit soft-memory cap when fetching archives, aborting')
archive_by_digest = {}
result = {}
for config, values in metadata_by_config.items():
for value in values:
if not value.get('digest'):
continue
if not archive_by_digest.get(value.get('digest')):
archive = self.archiver.retrieve(value.get('digest'), value.get('size', None))
if not archive:
continue
archive_by_digest[value.get('digest')] = archive
archive_by_digest.get(value.get('digest')).seek(0)
result.setdefault(config, [])
result[config].append(dict(
archive=archive_by_digest.get(value.get('digest')),
digest=value.get('digest'),
uuid=value['uuid'],
start_time=value['start_time'],
))
return result
def init_database(self, redis=StrictRedis, cassandra=CassandraContext):
redis_instance = redis()
self.stash_repository = MockStashRepository.safari(redis_instance)
self.svn_repository = MockSVNRepository.webkit(redis_instance)
cassandra.drop_keyspace(keyspace=self.KEYSPACE)
self.database = CommitContext(
redis=redis_instance,
cassandra=cassandra(keyspace=self.KEYSPACE, create_keyspace=True),
)
self.database.register_repository(self.stash_repository)
self.database.register_repository(self.svn_repository)
def __init__(self,
redis,
cassandra,
repositories=[],
default_ttl_seconds=TTL_YEAR * 5,
async_processing=False):
if default_ttl_seconds is not None and default_ttl_seconds < 4 * self.TTL_WEEK:
raise ValueError('TTL must be at least 4 weeks')
self.default_ttl_seconds = default_ttl_seconds
self._async_processing = async_processing
self.redis = redis
self.cassandra = cassandra
self.commit_context = CommitContext(redis, cassandra)
for repository in repositories:
self.commit_context.register_repository(repository)
self.configuration_context = ConfigurationContext(redis, cassandra)
self.upload_context = UploadContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
async_processing=async_processing,
)
self.suite_context = SuiteContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
self.test_context = TestContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
self.ci_context = CIContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
for context in [
self.suite_context, self.test_context, self.ci_context
]:
self.upload_context.register_upload_callback(
context.name, context.register)
self.archive_context = ArchiveContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
def init_database(self, redis=StrictRedis, cassandra=CassandraContext):
redis_instance = redis()
self.stash_repository = StashRepository(
'https://bitbucket.example.com/projects/SAFARI/repos/safari')
self.svn_repository = WebKitRepository()
cassandra.drop_keyspace(keyspace=self.KEYSPACE)
self.database = CommitContext(
redis=redis_instance,
cassandra=cassandra(keyspace=self.KEYSPACE, create_keyspace=True),
)
self.database.register_repository(self.stash_repository)
self.database.register_repository(self.svn_repository)
def test_uuid_for_commits(self,
redis=StrictRedis,
cassandra=CassandraContext):
with MockModelFactory.safari(), MockModelFactory.webkit():
self.init_database(redis=redis, cassandra=cassandra)
uuid = CommitContext.uuid_for_commits([
self.stash_repository.commit(ref='bae5d1e90999'),
self.svn_repository.commit(ref=6),
])
self.assertEqual(uuid, 160166800000)
def find_test_results(self,
configurations,
suite,
branch=None,
begin=None,
end=None,
recent=True,
limit=100):
if not isinstance(suite, str):
raise TypeError(f'Expected type {str}, got {type(suite)}')
with self:
result = {}
for configuration in configurations:
# FIXME: Remove the UploadsByConfigurationLegacy once results in it are sufficently old in Spring 2021
# We don't need to ignore duplicates because we never reported to both databases
result.update({
config: [value.unpack() for value in values]
for config, values in self.configuration_context.
select_from_table_with_configurations(
self.UploadsByConfigurationLegacy.__table_name__,
configurations=[configuration],
recent=recent,
suite=suite,
sdk=configuration.sdk,
branch=branch
or self.commit_context.DEFAULT_BRANCH_KEY,
uuid__gte=CommitContext.convert_to_uuid(begin),
uuid__lte=CommitContext.convert_to_uuid(
end, CommitContext.timestamp_to_uuid()),
limit=limit,
).items()
})
result.update({
config: [value.unpack() for value in values]
for config, values in self.configuration_context.
select_from_table_with_configurations(
self.UploadsByConfiguration.__table_name__,
configurations=[configuration],
recent=recent,
suite=suite,
sdk=configuration.sdk,
branch=branch
or self.commit_context.DEFAULT_BRANCH_KEY,
uuid__gte=CommitContext.convert_to_uuid(begin),
uuid__lte=CommitContext.convert_to_uuid(
end, CommitContext.timestamp_to_uuid()),
limit=limit,
).items()
})
return result
def test_verify_table(self, redis=StrictRedis, cassandra=CassandraContext):
with MockModelFactory.safari(), MockModelFactory.webkit():
self.init_database(redis=redis, cassandra=cassandra)
CommitContext(redis=redis(),
cassandra=cassandra(keyspace=self.KEYSPACE))
class CommitContextTest(WaitForDockerTestCase):
KEYSPACE = 'commit_mapping_test_keyspace'
def init_database(self, redis=StrictRedis, cassandra=CassandraContext):
redis_instance = redis()
self.stash_repository = StashRepository(
'https://bitbucket.example.com/projects/SAFARI/repos/safari')
self.svn_repository = WebKitRepository()
cassandra.drop_keyspace(keyspace=self.KEYSPACE)
self.database = CommitContext(
redis=redis_instance,
cassandra=cassandra(keyspace=self.KEYSPACE, create_keyspace=True),
)
self.database.register_repository(self.stash_repository)
self.database.register_repository(self.svn_repository)
def add_all_commits_to_database(self):
with MockModelFactory.safari() as safari, MockModelFactory.webkit(
) as webkit:
with self.database, self.database.cassandra.batch_query_context():
for key, repository in dict(safari=safari,
webkit=webkit).items():
for branch, commits in repository.commits.items():
for commit in commits:
self.database.register_partial_commit(
key,
hash=commit.hash,
revision=commit.revision,
fast=False,
)
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_verify_table(self, redis=StrictRedis, cassandra=CassandraContext):
with MockModelFactory.safari(), MockModelFactory.webkit():
self.init_database(redis=redis, cassandra=cassandra)
CommitContext(redis=redis(),
cassandra=cassandra(keyspace=self.KEYSPACE))
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_commit_by_id(self, redis=StrictRedis, cassandra=CassandraContext):
with MockModelFactory.safari(), MockModelFactory.webkit():
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
self.assertEqual(
[self.stash_repository.commit(ref='d8bce26fa65c')],
self.database.find_commits_by_ref(repository_id='safari',
ref='d8bce26fa65c'),
)
self.assertEqual(
[self.svn_repository.commit(ref=6)],
self.database.find_commits_by_ref(repository_id='webkit',
ref=6),
)
self.assertEqual(
0,
len(
self.database.find_commits_by_ref(repository_id='webkit',
ref='1234')))
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_commit_by_uuid(self,
redis=StrictRedis,
cassandra=CassandraContext):
with MockModelFactory.safari(), MockModelFactory.webkit():
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
self.assertEqual(
[self.stash_repository.commit(ref='1abe25b443e9')],
self.database.find_commits_by_uuid(repository_id='safari',
branch='main',
uuid=160166300000),
)
self.assertEqual(
[self.svn_repository.commit(ref=6)],
self.database.find_commits_by_uuid(repository_id='webkit',
branch='main',
uuid=160163990000),
)
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_commit_by_timestamp(self,
redis=StrictRedis,
cassandra=CassandraContext):
with MockModelFactory.safari(), MockModelFactory.webkit():
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
self.assertEqual(
[self.stash_repository.commit(ref='1abe25b443e9')],
self.database.find_commits_by_timestamp(repository_id='safari',
branch='main',
timestamp=1601663000),
)
self.assertEqual(
[self.svn_repository.commit(ref=6)],
self.database.find_commits_by_timestamp(repository_id='webkit',
branch='main',
timestamp=1601639900),
)
self.assertEqual(
2,
len(
self.database.find_commits_by_timestamp(
repository_id='safari',
branch='main',
timestamp=1601668000)))
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_all_commits_stash(self,
redis=StrictRedis,
cassandra=CassandraContext):
with MockModelFactory.safari(), MockModelFactory.webkit():
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
self.assertEqual(
5,
len(
self.database.find_commits_in_range(repository_id='safari',
branch='main')))
self.assertEqual(
[
self.stash_repository.commit(ref='d8bce26fa65c'),
self.stash_repository.commit(ref='bae5d1e90999')
],
self.database.find_commits_in_range(repository_id='safari',
branch='main',
limit=2),
)
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_all_commits_svn(self,
redis=StrictRedis,
cassandra=CassandraContext):
with MockModelFactory.safari(), MockModelFactory.webkit():
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
self.assertEqual(
4,
len(
self.database.find_commits_in_range(repository_id='webkit',
branch='main')))
self.assertEqual(
[
self.svn_repository.commit(ref=6),
self.svn_repository.commit(ref=4)
],
self.database.find_commits_in_range(repository_id='webkit',
branch='main',
limit=2),
)
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_stash_commits_in_range(self,
redis=StrictRedis,
cassandra=CassandraContext):
with MockModelFactory.safari(), MockModelFactory.webkit():
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
self.assertEqual(
[
self.stash_repository.commit(ref='bae5d1e90999'),
self.stash_repository.commit(ref='1abe25b443e9')
],
self.database.find_commits_in_range(repository_id='safari',
branch='main',
begin=1601663000,
end=1601668000),
)
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_svn_commits_in_range(self,
redis=StrictRedis,
cassandra=CassandraContext):
with MockModelFactory.safari(), MockModelFactory.webkit():
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
self.assertEqual(
[
self.svn_repository.commit(ref=6),
self.svn_repository.commit(ref=4)
],
self.database.find_commits_in_range(repository_id='webkit',
branch='main',
begin=1601637900,
end=1601639900),
)
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_stash_commits_between(self,
redis=StrictRedis,
cassandra=CassandraContext):
with MockModelFactory.safari(), MockModelFactory.webkit():
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
commits = [
self.stash_repository.commit(ref='1abe25b443e9'),
self.stash_repository.commit(ref='fff83bb2d917'),
self.stash_repository.commit(ref='9b8311f25a77'),
]
self.assertEqual(
commits,
self.database.find_commits_in_range(repository_id='safari',
branch='main',
begin=commits[-1],
end=commits[0]))
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_svn_commits_between(self,
redis=StrictRedis,
cassandra=CassandraContext):
with MockModelFactory.safari(), MockModelFactory.webkit():
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
commits = [
self.svn_repository.commit(ref=6),
self.svn_repository.commit(ref=4),
self.svn_repository.commit(ref=2),
]
self.assertEqual(
commits,
self.database.find_commits_in_range(repository_id='webkit',
branch='main',
begin=commits[-1],
end=commits[0]))
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_commit_from_stash_repo(self,
redis=StrictRedis,
cassandra=CassandraContext):
with MockModelFactory.safari(), MockModelFactory.webkit():
self.init_database(redis=redis, cassandra=cassandra)
self.database.register_partial_commit('safari',
hash='d8bce26fa65c',
fast=False)
self.assertEqual(
[self.stash_repository.commit(ref='d8bce26fa65c')],
self.database.find_commits_by_ref(repository_id='safari',
ref='d8bce26fa65c'),
)
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_commit_from_svn_repo(self,
redis=StrictRedis,
cassandra=CassandraContext):
with MockModelFactory.safari(), MockModelFactory.webkit():
self.init_database(redis=redis, cassandra=cassandra)
self.database.register_partial_commit('webkit',
revision=6,
fast=False)
self.assertEqual(
[self.svn_repository.commit(ref=6)],
self.database.find_commits_by_ref(repository_id='webkit',
ref=6),
)
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_branches(self, redis=StrictRedis, cassandra=CassandraContext):
with MockModelFactory.safari(), MockModelFactory.webkit():
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
self.assertEqual(['branch-a', 'branch-b', 'main'],
self.database.branches(repository_id='safari'))
self.assertEqual(
['branch-a', 'branch-b', 'main'],
self.database.branches(repository_id='webkit'),
)
self.assertEqual(['branch-a', 'branch-b'],
self.database.branches(repository_id='safari',
branch='branch'))
self.assertEqual(['branch-a'],
self.database.branches(repository_id='webkit',
branch='branch-a'))
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_next_commit(self, redis=StrictRedis, cassandra=CassandraContext):
with MockModelFactory.safari(), MockModelFactory.webkit():
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
self.assertEqual(
self.database.next_commit(self.svn_repository.commit(ref=4)),
self.svn_repository.commit(ref=6),
)
self.assertEqual(
self.database.next_commit(
self.stash_repository.commit(ref='bae5d1e90999')),
self.stash_repository.commit(ref='d8bce26fa65c'),
)
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_previous_commit(self,
redis=StrictRedis,
cassandra=CassandraContext):
with MockModelFactory.safari(), MockModelFactory.webkit():
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
self.assertEqual(
self.svn_repository.commit(ref=4),
self.database.previous_commit(
self.svn_repository.commit(ref=6)),
)
self.assertEqual(
self.stash_repository.commit(ref='bae5d1e90999'),
self.database.previous_commit(
self.stash_repository.commit(ref='d8bce26fa65c')),
)
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_sibling_commits(self,
redis=StrictRedis,
cassandra=CassandraContext):
self.maxDiff = None
with MockModelFactory.safari(), MockModelFactory.webkit():
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
self.assertEqual(
self.database.sibling_commits(
self.svn_repository.commit(ref=6), ['safari']),
{
'safari': [
self.stash_repository.commit(ref='d8bce26fa65c'),
self.stash_repository.commit(ref='bae5d1e90999'),
self.stash_repository.commit(ref='1abe25b443e9'),
self.stash_repository.commit(ref='fff83bb2d917'),
self.stash_repository.commit(ref='9b8311f25a77'),
]
},
)
self.assertEqual(
self.database.sibling_commits(
self.stash_repository.commit(ref='d8bce26fa65c'),
['webkit']),
{'webkit': [self.svn_repository.commit(ref=6)]},
)
self.assertEqual(
self.database.sibling_commits(
self.svn_repository.commit(ref=1), ['safari']),
{'safari': []},
)
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_uuid_for_commits(self,
redis=StrictRedis,
cassandra=CassandraContext):
with MockModelFactory.safari(), MockModelFactory.webkit():
self.init_database(redis=redis, cassandra=cassandra)
uuid = CommitContext.uuid_for_commits([
self.stash_repository.commit(ref='bae5d1e90999'),
self.svn_repository.commit(ref=6),
])
self.assertEqual(uuid, 160166800000)
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_branch_keys_for_commits(self,
redis=StrictRedis,
cassandra=CassandraContext):
with MockModelFactory.safari(), MockModelFactory.webkit():
self.init_database(redis=redis, cassandra=cassandra)
branches = self.database.branch_keys_for_commits([
self.stash_repository.commit(ref='d8bce26fa65c'),
self.svn_repository.commit(ref=6),
])
self.assertEqual(branches, ['default'])
branches = self.database.branch_keys_for_commits([
self.stash_repository.commit(ref='621652add7fc'),
self.svn_repository.commit(ref=7),
])
self.assertEqual(branches, ['branch-a'])
branches = self.database.branch_keys_for_commits([
self.stash_repository.commit(ref='790725a6d79e'),
self.svn_repository.commit(ref=8),
])
self.assertEqual(branches, ['branch-b'])
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_commit_url(self, redis=StrictRedis, cassandra=CassandraContext):
with MockModelFactory.safari(), MockModelFactory.webkit():
self.init_database(redis=redis, cassandra=cassandra)
self.assertEqual(
'https://bitbucket.example.com/projects/SAFARI/repos/safari/commits/d8bce26fa65c6fc8f39c17927abb77f69fab82fc',
self.database.url(
self.stash_repository.commit(ref='d8bce26fa65c')),
)
self.assertEqual(
'https://commits.webkit.org/4@main',
self.database.url(self.svn_repository.commit(ref=6)),
)
class Model(object):
TTL_DAY = 60 * 60 * 24
TTL_WEEK = 7 * TTL_DAY
TTL_YEAR = 365 * TTL_DAY
def __init__(self,
redis,
cassandra,
repositories=[],
default_ttl_seconds=TTL_YEAR * 5,
async_processing=False):
if default_ttl_seconds is not None and default_ttl_seconds < 4 * self.TTL_WEEK:
raise ValueError('TTL must be at least 4 weeks')
self.default_ttl_seconds = default_ttl_seconds
self._async_processing = async_processing
self.redis = redis
self.cassandra = cassandra
self.commit_context = CommitContext(redis, cassandra)
for repository in repositories:
self.commit_context.register_repository(repository)
self.configuration_context = ConfigurationContext(redis, cassandra)
self.upload_context = UploadContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
async_processing=async_processing,
)
self.suite_context = SuiteContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
self.test_context = TestContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
self.ci_context = CIContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
for context in [
self.suite_context, self.test_context, self.ci_context
]:
self.upload_context.register_upload_callback(
context.name, context.register)
self.archive_context = ArchiveContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
def do_work(self):
if not self._async_processing:
raise RuntimeError(
'No work to be done, asynchronous processing disabled')
try:
self.upload_context.do_processing_work()
except Exception as e:
sys.stderr.write(f'{traceback.format_exc()}\n')
sys.stderr.write(f'{e}\n')
def test_verify_table(self, redis=StrictRedis, cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
CommitContext(redis=redis(),
cassandra=cassandra(keyspace=self.KEYSPACE))
def test_uuid_for_commits(self):
uuid = CommitContext.uuid_for_commits([
MockStashRepository.safari().commit_for_id(id='bb6bda5f'),
MockSVNRepository.webkit().commit_for_id(id=236544)
])
self.assertEqual(uuid, 153805240800)
class CommitContextTest(WaitForDockerTestCase):
KEYSPACE = 'commit_mapping_test_keyspace'
def init_database(self, redis=StrictRedis, cassandra=CassandraContext):
redis_instance = redis()
self.stash_repository = MockStashRepository.safari(redis_instance)
self.svn_repository = MockSVNRepository.webkit(redis_instance)
cassandra.drop_keyspace(keyspace=self.KEYSPACE)
self.database = CommitContext(
redis=redis_instance,
cassandra=cassandra(keyspace=self.KEYSPACE, create_keyspace=True),
)
self.database.register_repository(self.stash_repository)
self.database.register_repository(self.svn_repository)
def add_all_commits_to_database(self):
for mock_repository in [self.stash_repository, self.svn_repository]:
for commit_list in mock_repository.commits.values():
for commit in commit_list:
self.database.register_commit(commit)
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_verify_table(self, redis=StrictRedis, cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
CommitContext(redis=redis(),
cassandra=cassandra(keyspace=self.KEYSPACE))
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_commit_by_id(self, redis=StrictRedis, cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
self.assertEqual(
[
self.stash_repository.commit_for_id(id='bb6bda5f44',
branch='master')
],
self.database.find_commits_by_id(repository_id='safari',
branch='master',
commit_id='bb6bda5f44'),
)
self.assertEqual(
2,
len(
self.database.find_commits_by_id(repository_id='safari',
branch='master',
commit_id='336610a')))
self.assertEqual(
[self.svn_repository.commit_for_id(id=236544, branch='trunk')],
self.database.find_commits_by_id(repository_id='webkit',
branch='trunk',
commit_id=236544),
)
self.assertEqual(
0,
len(
self.database.find_commits_by_id(repository_id='webkit',
branch='trunk',
commit_id='23654')))
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_commit_by_uuid(self,
redis=StrictRedis,
cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
self.assertEqual(
[
self.stash_repository.commit_for_id(id='7be4084258',
branch='master')
],
self.database.find_commits_by_uuid(repository_id='safari',
branch='master',
uuid=153755068501),
)
self.assertEqual(
[self.svn_repository.commit_for_id(id=236540, branch='trunk')],
self.database.find_commits_by_uuid(repository_id='webkit',
branch='trunk',
uuid=153802947900),
)
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_commit_by_timestamp(self,
redis=StrictRedis,
cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
self.assertEqual(
[
self.stash_repository.commit_for_id(id='336610a84f',
branch='master')
],
self.database.find_commits_by_timestamp(repository_id='safari',
branch='master',
timestamp=1537809818),
)
self.assertEqual(
[self.svn_repository.commit_for_id(id=236540, branch='trunk')],
self.database.find_commits_by_timestamp(repository_id='webkit',
branch='trunk',
timestamp=1538029479),
)
self.assertEqual(
2,
len(
self.database.find_commits_by_timestamp(repository_id='safari',
branch='master',
timestamp=1537550685)))
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_all_commits_stash(self,
redis=StrictRedis,
cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
self.assertEqual(
5,
len(
self.database.find_commits_in_range(repository_id='safari',
branch='master')))
self.assertEqual(
[
self.stash_repository.commit_for_id(id='bb6bda5f44',
branch='master'),
self.stash_repository.commit_for_id(id='336610a84f',
branch='master')
],
self.database.find_commits_in_range(repository_id='safari',
branch='master',
limit=2),
)
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_all_commits_svn(self,
redis=StrictRedis,
cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
self.assertEqual(
5,
len(
self.database.find_commits_in_range(repository_id='webkit',
branch='trunk')))
self.assertEqual(
[
self.svn_repository.commit_for_id(id=236544, branch='trunk'),
self.svn_repository.commit_for_id(id=236543, branch='trunk')
],
self.database.find_commits_in_range(repository_id='webkit',
branch='trunk',
limit=2),
)
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_stash_commits_in_range(self,
redis=StrictRedis,
cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
self.assertEqual(
[
self.stash_repository.commit_for_id(id='bb6bda5f44',
branch='master'),
self.stash_repository.commit_for_id(id='336610a84f',
branch='master')
],
self.database.find_commits_in_range(repository_id='safari',
branch='master',
begin=1537809818,
end=1537810281),
)
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_svn_commits_in_range(self,
redis=StrictRedis,
cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
self.assertEqual(
[
self.svn_repository.commit_for_id(id=236544, branch='trunk'),
self.svn_repository.commit_for_id(id=236543, branch='trunk')
],
self.database.find_commits_in_range(repository_id='webkit',
branch='trunk',
begin=1538050458,
end=1538052408),
)
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_stash_commits_between(self,
redis=StrictRedis,
cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
commits = [
self.stash_repository.commit_for_id(id='bb6bda5f',
branch='master'),
self.stash_repository.commit_for_id(id='336610a8',
branch='master'),
self.stash_repository.commit_for_id(id='336610a4',
branch='master'),
]
self.assertEqual(
commits,
self.database.find_commits_in_range(repository_id='safari',
branch='master',
begin=commits[-1],
end=commits[0]))
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_svn_commits_between(self,
redis=StrictRedis,
cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
commits = [
self.svn_repository.commit_for_id(id=236544, branch='trunk'),
self.svn_repository.commit_for_id(id=236543, branch='trunk'),
self.svn_repository.commit_for_id(id=236542, branch='trunk'),
]
self.assertEqual(
commits,
self.database.find_commits_in_range(repository_id='webkit',
branch='trunk',
begin=commits[-1],
end=commits[0]))
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_commit_from_stash_repo(self,
redis=StrictRedis,
cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
self.database.register_commit_with_repo_and_id('safari', 'master',
'bb6bda5f44')
self.assertEqual(
[
self.stash_repository.commit_for_id(id='bb6bda5f44',
branch='master')
],
self.database.find_commits_by_id(repository_id='safari',
branch='master',
commit_id='bb6bda5f44'),
)
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_commit_from_svn_repo(self,
redis=StrictRedis,
cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
self.database.register_commit_with_repo_and_id('webkit', 'trunk',
236544)
self.assertEqual(
[self.svn_repository.commit_for_id(id=236544, branch='trunk')],
self.database.find_commits_by_id(repository_id='webkit',
branch='trunk',
commit_id=236544),
)
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_branches(self, redis=StrictRedis, cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
self.assertEqual(['master', 'safari-606-branch'],
self.database.branches(repository_id='safari'))
self.assertEqual(['safari-606-branch', 'trunk'],
self.database.branches(repository_id='webkit'))
self.assertEqual(['safari-606-branch'],
self.database.branches(repository_id='safari',
branch='safari'))
self.assertEqual(['safari-606-branch'],
self.database.branches(repository_id='webkit',
branch='safari-606-branch'))
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_next_commit(self, redis=StrictRedis, cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
self.assertEqual(
self.database.next_commit(
self.svn_repository.commit_for_id(id=236542)),
self.svn_repository.commit_for_id(id=236543),
)
self.assertEqual(
self.database.next_commit(
self.stash_repository.commit_for_id(
id='336610a40c3fecb728871e12ca31482ca715b383')),
self.stash_repository.commit_for_id(
id='336610a84fdcf14ddcf1db65075af95480516fda'),
)
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_previous_commit(self,
redis=StrictRedis,
cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
self.assertEqual(
self.svn_repository.commit_for_id(id=236542),
self.database.previous_commit(
self.svn_repository.commit_for_id(id=236543)),
)
self.assertEqual(
self.stash_repository.commit_for_id(
id='336610a40c3fecb728871e12ca31482ca715b383'),
self.database.previous_commit(
self.stash_repository.commit_for_id(
id='336610a84fdcf14ddcf1db65075af95480516fda')),
)
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_sibling_commits(self,
redis=StrictRedis,
cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
self.add_all_commits_to_database()
self.assertEqual(
self.database.sibling_commits(
self.svn_repository.commit_for_id(id=236542), ['safari']),
{
'safari': [
self.stash_repository.commit_for_id(
id='bb6bda5f44dd24d0b54539b8ff6e8c17f519249a')
]
},
)
self.assertEqual(
self.database.sibling_commits(
self.stash_repository.commit_for_id(
id='bb6bda5f44dd24d0b54539b8ff6e8c17f519249a'),
['webkit']),
{
'webkit': [
self.svn_repository.commit_for_id(id=236544),
self.svn_repository.commit_for_id(id=236543),
self.svn_repository.commit_for_id(id=236542),
self.svn_repository.commit_for_id(id=236541),
self.svn_repository.commit_for_id(id=236540),
]
},
)
self.assertEqual(
self.database.sibling_commits(
self.stash_repository.commit_for_id(
id='336610a84fdcf14ddcf1db65075af95480516fda'),
['webkit']),
{'webkit': []},
)
def test_uuid_for_commits(self):
uuid = CommitContext.uuid_for_commits([
MockStashRepository.safari().commit_for_id(id='bb6bda5f'),
MockSVNRepository.webkit().commit_for_id(id=236544)
])
self.assertEqual(uuid, 153805240800)
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_branch_keys_for_commits(self,
redis=StrictRedis,
cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
branches = self.database.branch_keys_for_commits([
MockStashRepository.safari().commit_for_id(id='bb6bda5f'),
MockSVNRepository.webkit().commit_for_id(id=236544),
])
self.assertEqual(branches, ['default'])
branches = self.database.branch_keys_for_commits([
MockStashRepository.safari().commit_for_id(
id='79256c32', branch='safari-606-branch'),
MockSVNRepository.webkit().commit_for_id(id=236544),
])
self.assertEqual(branches, ['safari-606-branch'])
branches = self.database.branch_keys_for_commits([
MockStashRepository.safari().commit_for_id(
id='79256c32', branch='safari-606-branch'),
MockSVNRepository.webkit().commit_for_id(
id=236335, branch='safari-606-branch'),
])
self.assertEqual(branches, ['safari-606-branch'])
@WaitForDockerTestCase.mock_if_no_docker(
mock_redis=FakeStrictRedis, mock_cassandra=MockCassandraContext)
def test_commit_url(self, redis=StrictRedis, cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
self.assertEqual(
'https://fake-stash-instance.apple.com/projects/BROWSER/repos/safari/commits/bb6bda5f44dd24d0b54539b8ff6e8c17f519249a',
self.database.url(
MockStashRepository.safari().commit_for_id(id='bb6bda5f')),
)
self.assertEqual(
'https://trac.webkit.org/changeset/236544/webkit',
self.database.url(
MockSVNRepository.webkit().commit_for_id(id=236544)),
)
def _failures(
self,
all_table,
unexpected_table,
configurations,
suite,
recent=True,
branch=None,
begin=None,
end=None,
begin_query_time=None,
end_query_time=None,
unexpected=True,
collapsed=True,
limit=DEFAULT_LIMIT,
):
table = unexpected_table if unexpected else all_table
if not isinstance(suite, str):
raise TypeError(
f'Expected type {str} for suite, got {type(suite)}')
def get_time(time):
if isinstance(time, datetime):
return time
elif time:
return datetime.utcfromtimestamp(int(time))
return None
with self:
has_test_runs = False
if collapsed:
result = set()
else:
result = {}
for configuration in configurations:
for config, values in self.configuration_context.select_from_table_with_configurations(
table.__table_name__,
configurations=[configuration],
recent=recent,
suite=suite,
sdk=configuration.sdk,
branch=branch
or self.commit_context.DEFAULT_BRANCH_KEY,
uuid__gte=CommitContext.convert_to_uuid(begin),
uuid__lte=CommitContext.convert_to_uuid(
end, CommitContext.timestamp_to_uuid()),
start_time__gte=get_time(begin_query_time),
start_time__lte=get_time(end_query_time),
limit=limit,
).items():
if collapsed:
for value in values:
has_test_runs = True
for test in value.unpack():
if test not in ['uuid', 'start_time']:
result.add(test)
else:
runs = []
for value in values:
has_test_runs = True
# uuid and start_time are not in the unpacked values
unpacked = value.unpack()
if len(unpacked) > 2:
runs.append(unpacked)
if runs:
result.update({config: runs})
return result if has_test_runs else None
class Model(object):
TTL_DAY = 60 * 60 * 24
TTL_WEEK = 7 * TTL_DAY
TTL_YEAR = 365 * TTL_DAY
class HealthTable(CassandraModel):
__table_name__ = 'health'
key = columns.Text(partition_key=True, required=True)
value = columns.Text(required=True)
def __init__(self,
redis,
cassandra,
repositories=[],
default_ttl_seconds=TTL_YEAR * 5,
archive_ttl_seconds=TTL_WEEK * 8,
async_processing=False,
s3_credentials=None):
if default_ttl_seconds is not None and default_ttl_seconds < 4 * self.TTL_WEEK:
raise ValueError('TTL must be at least 4 weeks')
if archive_ttl_seconds is not None and archive_ttl_seconds < 2 * self.TTL_WEEK:
raise ValueError('Archive TTL must be at least 2 weeks')
self.default_ttl_seconds = default_ttl_seconds
self.archive_ttl_seconds = archive_ttl_seconds or default_ttl_seconds
self._async_processing = async_processing
self.redis = redis
self.cassandra = cassandra
with self.cassandra:
self.cassandra.create_table(self.HealthTable)
self.commit_context = CommitContext(redis, cassandra)
for repository in repositories:
self.commit_context.register_repository(repository)
self.configuration_context = ConfigurationContext(redis, cassandra)
self.upload_context = UploadContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
async_processing=async_processing,
)
self.suite_context = SuiteContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
self.test_context = TestContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
self.failure_context = FailureContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
self.ci_context = CIContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.default_ttl_seconds,
)
for context in [
self.ci_context, self.failure_context, self.suite_context,
self.test_context, self.commit_context
]:
self.upload_context.register_upload_callback(
context.name, context.register)
self.archive_context = ArchiveContext(
configuration_context=self.configuration_context,
commit_context=self.commit_context,
ttl_seconds=self.archive_ttl_seconds,
s3_credentials=s3_credentials,
)
def healthy(self, writable=True):
if writable:
self.redis.set('health-check', b'healthy', ex=5 * 60)
value = self.redis.get('health-check')
if value is not None and value != b'healthy':
return False
with self.cassandra:
if writable:
self.cassandra.insert_row(
self.HealthTable.__table_name__,
key='health-check',
value='healthy',
ttl=5 * 60,
)
values = [
element.value for element in self.cassandra.select_from_table(
self.HealthTable.__table_name__,
limit=100,
key='health-check',
)
]
if values and values != ['healthy']:
return False
return True
def do_work(self):
if not self._async_processing:
raise RuntimeError(
'No work to be done, asynchronous processing disabled')
try:
return self.upload_context.do_processing_work()
except Exception as e:
sys.stderr.write(f'{traceback.format_exc()}\n')
sys.stderr.write(f'{e}\n')
return False
def find_archive(
self, configurations=None, suite=None, recent=True,
branch=None, begin=None, end=None,
begin_query_time=None, end_query_time=None,
limit=DEFAULT_LIMIT,
):
if not configurations:
configurations = []
if not isinstance(suite, str):
raise TypeError(f'Expected type {str}, got {type(suite)}')
with self:
metadata_by_config = {}
for configuration in configurations:
metadata_by_config.update({config: [value.unpack() for value in values] for config, values in self.configuration_context.select_from_table_with_configurations(
self.ArchiveMetaDataByCommit.__table_name__, configurations=[configuration], recent=recent,
suite=suite, sdk=configuration.sdk, branch=branch or self.commit_context.DEFAULT_BRANCH_KEY,
uuid__gte=CommitContext.convert_to_uuid(begin),
uuid__lte=CommitContext.convert_to_uuid(end, CommitContext.timestamp_to_uuid()),
start_time__gte=_get_time(begin_query_time), start_time__lte=_get_time(end_query_time),
limit=limit,
).items()})
memory_used = 0
for values in metadata_by_config.values():
for value in values:
if not value.get('digest'):
continue
memory_used += value.get('size', 0)
if memory_used > self.MEMORY_LIMIT:
raise RuntimeError('Hit soft-memory cap when fetching archives, aborting')
result = {}
for config, values in metadata_by_config.items():
for value in values:
if not value.get('digest'):
continue
rows = self.cassandra.select_from_table(
self.ArchiveChunks.__table_name__,
digest=value.get('digest'),
limit=1 + int(value.get('size', 0) / self.CHUNK_SIZE),
)
if len(rows) == 0:
continue
digest = hashlib.md5()
archive = io.BytesIO()
archive_size = 0
for row in rows:
archive_size += len(row.chunk)
digest.update(row.chunk)
archive.write(row.chunk)
if archive_size != value.get('size', 0) or value.get('digest', '') != digest.hexdigest():
raise RuntimeError('Failed to reconstruct archive from chunks')
archive.seek(0)
result.setdefault(config, [])
result[config].append(dict(
archive=archive,
uuid=value['uuid'],
start_time=value['start_time'],
))
return result
