def suites(self,
configurations=None,
recent=None,
suite=None,
branch=None,
**kwargs):
AssertRequest.is_type(['GET'])
AssertRequest.query_kwargs_empty(**kwargs)
with self.upload_context:
suites_by_config = self.upload_context.find_suites(
configurations=configurations,
recent=boolean_query(*recent)[0] if recent else True,
branch=branch[0] if branch else None,
)
result = []
for config, candidate_suites in suites_by_config.items():
suites_for_config = [
s for s in candidate_suites if not suite or s in suite
]
if suites_for_config:
result.append([config, suites_for_config])
if not result:
abort(404,
description='No suites matching the specified criteria')
return jsonify(Configuration.Encoder().default(result))
def add_mock_archives(cls,
model,
configuration=Configuration(),
suite='layout-tests',
archive=None):
archive = archive or io.BytesIO(base64.b64decode(cls.ARCHIVE_ZIP))
configurations = [configuration] if configuration.is_complete(
) else ConfigurationContextTest.CONFIGURATIONS
with model.upload_context:
current = time.time()
old = current - cls.THREE_WEEKS
for complete_configuration in configurations:
if complete_configuration != configuration:
continue
timestamp_to_use = current
if (complete_configuration.platform == 'Mac' and complete_configuration.version <= Configuration.version_to_integer('10.13')) \
or (complete_configuration.platform == 'iOS' and complete_configuration.version <= Configuration.version_to_integer('11')):
timestamp_to_use = old
cls.iterate_all_commits(
model, lambda commits: model.archive_context.register(
archive,
complete_configuration,
commits,
suite=suite,
timestamp=timestamp_to_use))
def process_test_results(self, configuration, commits, suite, test_results, timestamp=None):
timestamp = timestamp or time.time()
if not self._async_processing:
return self.synchronously_process_test_results(configuration, commits, suite, test_results=test_results, timestamp=timestamp)
for branch in self.commit_context.branch_keys_for_commits(commits):
hash_key = hash(configuration) ^ hash(branch) ^ hash(self.commit_context.uuid_for_commits(commits)) ^ hash(
suite)
self.redis.set(
f'{self.QUEUE_NAME}:{hash_key}',
json.dumps(dict(started_processing=0, attempts=0)),
ex=self.PROCESS_TIMEOUT,
)
self.redis.set(
f'data_for_{self.QUEUE_NAME}:{hash_key}',
json.dumps(dict(
configuration=Configuration.Encoder().default(configuration),
suite=suite,
commits=Commit.Encoder().default(commits),
timestamp=timestamp,
test_results=test_results,
)),
ex=self.PROCESS_TIMEOUT,
)
return {key: dict(status='Queued') for key in list(self._process_upload_callbacks[suite].keys()) + list(self._process_upload_callbacks[None].keys())}
def test_suite_list(self, redis=StrictRedis, cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
MockModelFactory.add_mock_results(self.model)
for suites in self.model.upload_context.find_suites(configurations=[Configuration()], recent=True).values():
self.assertEqual(suites, ['layout-tests'])
MockModelFactory.add_mock_results(self.model, suite='api_tests')
for suites in self.model.upload_context.find_suites(configurations=[Configuration()], recent=True).values():
self.assertEqual(suites, ['api_tests', 'layout-tests'])
MockModelFactory.add_mock_results(self.model, configuration=Configuration(is_simulator=True), suite='webkitpy')
for suites in self.model.upload_context.find_suites(configurations=[Configuration(is_simulator=True)], recent=True).values():
self.assertEqual(suites, ['api_tests', 'layout-tests', 'webkitpy'])
for suites in self.model.upload_context.find_suites(configurations=[Configuration(is_simulator=False)], recent=True).values():
self.assertEqual(suites, ['api_tests', 'layout-tests'])
def download(self):
AssertRequest.is_type(['GET'])
with self.upload_context:
uploads = self._find_uploads_for_query()
response = []
for config, suite_results in uploads.items():
for suite, results in suite_results.items():
for result in results:
config.sdk = result.get('sdk')
response.append(
dict(
configuration=Configuration.Encoder().default(
config),
suite=suite,
commits=[
commit.Encoder().default(commit)
for commit in result['commits']
],
timestamp=result['timestamp'],
test_results=result['test_results'],
))
return jsonify(response)
def add_mock_results(cls,
model,
configuration=Configuration(),
suite='layout-tests',
test_results=None):
if test_results is None:
test_results = cls.layout_test_results()
configurations = [configuration] if configuration.is_complete(
) else ConfigurationContextTest.CONFIGURATIONS
with model.upload_context:
current = time.time()
old = current - 60 * 60 * 24 * 21
for complete_configuration in configurations:
if complete_configuration != configuration:
continue
timestamp_to_use = current
if (complete_configuration.platform == 'Mac' and complete_configuration.version <= Configuration.version_to_integer('10.13')) \
or (complete_configuration.platform == 'iOS' and complete_configuration.version <= Configuration.version_to_integer('11')):
timestamp_to_use = old
cls.iterate_all_commits(
model, lambda commits: model.upload_context.
upload_test_results(complete_configuration,
commits,
suite=suite,
test_results=test_results,
timestamp=timestamp_to_use))
def test_invalid_configuration(self, redis=StrictRedis, cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
with self.assertRaises(TypeError):
self.database.register_configuration('invalid object')
with self.assertRaises(TypeError):
self.database.register_configuration(Configuration(platform='iOS'))
def process(self):
AssertRequest.is_type(['POST'])
with self.upload_context:
uploads = self._find_uploads_for_query()
if not uploads:
abort(404,
description='No uploads matching the specified criteria')
response = []
for config, suite_results in uploads.items():
for suite, results in suite_results.items():
for result in results:
config.sdk = result.get('sdk')
processing_results = self.upload_context.process_test_results(
configuration=config,
commits=result['commits'],
suite=suite,
test_results=result['test_results'],
timestamp=result['timestamp'],
)
response.append(
dict(
configuration=Configuration.Encoder().default(
config),
suite=suite,
commits=Commit.Encoder().default(
result['commits']),
timestamp=result['timestamp'],
processing=processing_results,
))
return jsonify(response)
def process_results(self,
model,
configuration=Configuration(),
suite='layout-tests'):
configurations = [configuration] if configuration.is_complete(
) else ConfigurationContextTest.CONFIGURATIONS
with model.upload_context:
for complete_configuration in configurations:
if complete_configuration != configuration:
continue
for branch in (None, 'safari-606-branch'):
results_dict = model.upload_context.find_test_results(
configurations=[complete_configuration],
suite=suite,
branch=branch,
recent=False,
)
for config, results in results_dict.items():
for result in results:
model.upload_context.process_test_results(
configuration=config,
commits=result['commits'],
suite=suite,
test_results=result['test_results'],
timestamp=result['timestamp'],
)
def to_configuration(self):
return Configuration(platform=self.platform,
version=self.version,
is_simulator=self.is_simulator,
architecture=self.architecture,
sdk=getattr(self, 'sdk', None) or None,
**json.loads(self.attributes))
def test_sdk_differentiation(self, redis=StrictRedis, cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
MockModelFactory.add_mock_results(self.model)
configuration_to_search = Configuration(platform='iOS', version='12.0.0', is_simulator=True, style='Asan')
results = self.model.upload_context.find_test_results(configurations=[configuration_to_search], suite='layout-tests', recent=False)
self.assertEqual(1, len(results))
MockModelFactory.add_mock_results(self.model, configuration=Configuration(
platform='iOS', version='12.0.0', sdk='16A405', is_simulator=True, architecture='x86_64', style='Asan',
))
results = self.model.upload_context.find_test_results(configurations=[configuration_to_search], suite='layout-tests', recent=False)
self.assertEqual(2, len(results))
results = self.model.upload_context.find_test_results(configurations=[Configuration(platform='iOS', sdk='16A405')], suite='layout-tests', recent=False)
self.assertEqual(1, len(results))
def test_all_successful(self,
redis=StrictRedis,
cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
results = self.model.suite_context.find_by_start_time(
configurations=[
Configuration(platform='Mac', style='Release', flavor='wk1')
],
suite='layout-tests',
begin=MockSVNRepository.webkit().commit_for_id(236542),
end=MockSVNRepository.webkit().commit_for_id(236542),
)
self.assertEqual(
next(iter(results.values()))[0]['stats'],
dict(
tests_run=4,
tests_skipped=0,
tests_failed=0,
tests_timedout=0,
tests_crashed=0,
tests_unexpected_failed=0,
tests_unexpected_timedout=0,
tests_unexpected_crashed=0,
),
)
def select_from_table_with_configurations(self,
table_name,
configurations=None,
branch=None,
recent=True,
limit=100,
**kwargs):
if not isinstance(configurations, Iterable):
raise TypeError('Expected configurations to be iterable')
if not configurations:
configurations.append(Configuration())
with self:
complete_configurations = set()
for config in configurations:
if not isinstance(config, Configuration):
raise TypeError(
f'Expected type {Configuration}, got {type(config)}')
if config.is_complete():
complete_configurations.add(config)
elif recent:
[
complete_configurations.add(element)
for element in self.search_for_recent_configuration(
config, branch=branch)
]
else:
[
complete_configurations.add(element) for element in
self.search_for_configuration(config, branch=branch)
]
results = {}
for configuration in complete_configurations:
attributes_dict = OrderedDict()
for member in Configuration.OPTIONAL_MEMBERS:
if getattr(configuration, member) is not None:
attributes_dict[member] = getattr(
configuration, member)
rows = self.cassandra.select_from_table(
table_name,
platform=configuration.platform,
is_simulator=configuration.is_simulator,
version=configuration.version,
architecture=configuration.architecture,
attributes=json.dumps(attributes_dict),
limit=limit,
branch=branch or CommitContext.DEFAULT_BRANCH_KEY,
**kwargs)
if len(rows) == 0:
continue
for row in rows:
full_config = row.to_configuration()
if not results.get(full_config):
results[full_config] = []
results[full_config].append(row)
return results
def _do_job_for_key(self, key, attempts=1):
job_complete = False
try:
raw_data = self.redis.get(f'data_for_{key}')
if raw_data:
data = json.loads(raw_data)
self.synchronously_process_test_results(
configuration=Configuration.from_json(
data['configuration']),
commits=[
Commit.from_json(commit_json)
for commit_json in data['commits']
],
suite=data['suite'],
timestamp=data['timestamp'],
test_results=data['test_results'],
)
job_complete = True
finally:
if job_complete or attempts >= self.MAX_ATTEMPTS:
self.redis.delete(key)
self.redis.delete(f'data_for_{key}')
else:
self.redis.set(
key,
json.dumps(dict(started_processing=0, attempts=attempts)),
ex=self.PROCESS_TIMEOUT,
)
def test_no_test_runs(self, redis=StrictRedis, cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
results = self.model.failure_context.failures_by_commit(
configurations=[Configuration(platform='Mac', style='Release', flavor='wk1')],
suite='layout-tests', recent=True, end=0,
)
self.assertEqual(results, None)
def test_configuration_by_model(self, redis=StrictRedis, cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
self.register_configurations()
configuration_to_search_for = Configuration(model='iPhone 8')
matching_configurations = self.database.search_for_configuration(configuration_to_search_for)
self.assertEqual(4, len(matching_configurations))
for config in matching_configurations:
self.assertEqual(configuration_to_search_for, config)
def search_for_recent_configuration(self,
configuration=Configuration(),
branch=None):
if not isinstance(configuration, Configuration):
raise TypeError(
f'Expected type {Configuration}, got {type(configuration)}')
configurations = []
for key in self.redis.scan_iter(
self._convert_to_redis_key(
configuration, branch
or CommitContext.DEFAULT_BRANCH_KEY)):
candidate = Configuration.from_json(
self.redis.get(key.decode('utf-8')).decode('utf-8'))
if candidate == configuration:
configurations.append(candidate)
return configurations
def test_result_retrieval(self,
redis=StrictRedis,
cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
MockModelFactory.add_mock_results(self.model)
results = self.model.upload_context.find_test_results(
configurations=[Configuration(platform='Mac')],
suite='layout-tests',
recent=True)
self.assertEqual(6, len(results))
for config, values in results.items():
self.assertEqual(config, Configuration(platform='Mac'))
self.assertEqual(5, len(values))
for value in values:
self.assertEqual(value['test_results'],
MockModelFactory.layout_test_results())
def test_recent_configurations_constrained(self, redis=StrictRedis, cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
self.register_configurations()
configuration_to_search_for = Configuration(architecture='arm64', style='Release')
matching_configurations = self.database.search_for_recent_configuration(configuration_to_search_for)
self.assertEqual(1, len(matching_configurations))
self.assertEqual(matching_configurations[0], configuration_to_search_for)
def find_run_results(self,
suite=None,
configurations=None,
recent=None,
branch=None,
begin=None,
end=None,
begin_query_time=None,
end_query_time=None,
limit=None,
**kwargs):
AssertRequest.is_type(['GET'])
AssertRequest.query_kwargs_empty(**kwargs)
recent = boolean_query(*recent)[0] if recent else True
with self.suite_context:
if not suite:
abort(400, description='No suite specified')
query_dict = dict(
suite=suite,
configurations=configurations,
recent=recent,
branch=branch[0],
begin=begin,
end=end,
begin_query_time=begin_query_time,
end_query_time=end_query_time,
limit=limit,
)
specified_commits = sum(
[1 if element else 0 for element in [begin, end]])
specified_timestamps = sum([
1 if element else 0
for element in [begin_query_time, end_query_time]
])
if specified_commits >= specified_timestamps:
find_function = self.suite_context.find_by_commit
def sort_function(result):
return result['uuid']
else:
find_function = self.suite_context.find_by_start_time
def sort_function(result):
return result['start_time']
response = []
for config, results in find_function(**query_dict).items():
response.append(
dict(
configuration=Configuration.Encoder().default(config),
results=sorted(results, key=sort_function),
))
return jsonify(response)
def _convert_to_redis_key(cls, configuration, branch):
return 'configs_with_branch:{}:{}:{}:{}:{}:{}:{}:{}:{}'.format(
configuration.platform or '*', '*' if configuration.is_simulator is None else (1 if configuration.is_simulator else 0),
'*' if configuration.version is None else Configuration.integer_to_version(configuration.version),
configuration.version_name or '*',
configuration.architecture or '*', configuration.model or '*',
configuration.style or '*', configuration.flavor or '*',
branch,
)
def test_recent_configurations_constrained_by_version(self, redis=StrictRedis, cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
self.register_configurations()
configuration_to_search_for = Configuration(version='12', is_simulator=True)
matching_configurations = self.database.search_for_recent_configuration(configuration_to_search_for)
self.assertEqual(3, len(matching_configurations))
for config in matching_configurations:
self.assertEqual(configuration_to_search_for, config)
def test_configuration_by_platform(self, redis=StrictRedis, cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
self.register_configurations()
configuration_to_search_for = Configuration(platform='Mac', style='Debug')
matching_configurations = self.database.search_for_configuration(configuration_to_search_for)
self.assertEqual(4, len(matching_configurations))
for config in matching_configurations:
self.assertEqual(configuration_to_search_for, config)
def test_configuration_by_architecture(self, redis=StrictRedis, cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
self.register_configurations()
configuration_to_search_for = Configuration(architecture='x86_64', style='Release')
matching_configurations = self.database.search_for_configuration(configuration_to_search_for)
self.assertEqual(6, len(matching_configurations))
for config in matching_configurations:
self.assertEqual(configuration_to_search_for, config)
def to_configuration(self):
return Configuration(
platform=self.platform,
version=self.version, version_name=self.version_name or None,
sdk=getattr(self, 'sdk', None) or None,
is_simulator=self.is_simulator,
architecture=self.architecture, model=self.model or None,
style=self.style or None, flavor=self.flavor or None,
)
def test_unexpected_failures_collapsed(self, redis=StrictRedis, cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
results = self.model.failure_context.failures_by_commit(
configurations=[Configuration(platform='Mac', style='Release', flavor='wk1')],
suite='layout-tests', recent=True,
)
self.assertEqual(len(results), 1)
self.assertEqual(results, set(['fast/encoding/css-cached-bom.html']))
def test_file_list(self, redis=StrictRedis, cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
files = self.model.archive_context.file(
configurations=[Configuration(platform='Mac', style='Release', flavor='wk1')],
begin=1601660000, end=1601660000,
suite='layout-tests',
)
self.assertEqual(len(next(iter(files.values()))), 1)
self.assertEqual(next(iter(files.values()))[0]['uuid'], 160166000000)
self.assertEqual(next(iter(files.values()))[0]['file'], ['file.txt', 'index.html'])
def test_find_archive(self, redis=StrictRedis, cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
archives = self.model.archive_context.find_archive(
configurations=[Configuration(platform='Mac', style='Release', flavor='wk1')],
begin=1601660000, end=1601660000,
suite='layout-tests',
)
self.assertEqual(len(next(iter(archives.values()))), 1)
self.assertEqual(next(iter(archives.values()))[0]['uuid'], 160166000000)
self.assertEqual(next(iter(archives.values()))[0]['archive'].getvalue(), base64.b64decode(MockModelFactory.ARCHIVE_ZIP))
def test_file_list(self, redis=StrictRedis, cassandra=CassandraContext):
self.init_database(redis=redis, cassandra=cassandra)
files = self.model.archive_context.file(
configurations=[Configuration(platform='Mac', style='Release', flavor='wk1')],
begin=MockSVNRepository.webkit().commit_for_id(236542), end=MockSVNRepository.webkit().commit_for_id(236542),
suite='layout-tests',
)
self.assertEqual(len(next(iter(files.values()))), 1)
self.assertEqual(next(iter(files.values()))[0]['uuid'], 153804910800)
self.assertEqual(next(iter(files.values()))[0]['file'], ['file.txt', 'index.html'])
def test_no_failures(self, redis=StrictRedis, cassandra=CassandraContext):
cassandra.drop_keyspace(keyspace=self.KEYSPACE)
self.model = MockModelFactory.create(redis=redis(), cassandra=cassandra(keyspace=self.KEYSPACE, create_keyspace=True))
MockModelFactory.add_mock_results(self.model)
MockModelFactory.process_results(self.model)
results = self.model.failure_context.failures_by_commit(
configurations=[Configuration(platform='Mac', style='Release', flavor='wk1')],
suite='layout-tests', recent=True, collapsed=False, unexpected=False,
)
self.assertEqual(len(results), 0)