def suites(self, configurations=None, recent=None, suite=None, branch=None, **kwargs): AssertRequest.is_type(['GET']) AssertRequest.query_kwargs_empty(**kwargs) with self.upload_context: suites_by_config = self.upload_context.find_suites( configurations=configurations, recent=boolean_query(*recent)[0] if recent else True, branch=branch[0] if branch else None, ) result = [] for config, candidate_suites in suites_by_config.items(): suites_for_config = [ s for s in candidate_suites if not suite or s in suite ] if suites_for_config: result.append([config, suites_for_config]) if not result: abort(404, description='No suites matching the specified criteria') return jsonify(Configuration.Encoder().default(result))
def add_mock_archives(cls, model, configuration=Configuration(), suite='layout-tests', archive=None): archive = archive or io.BytesIO(base64.b64decode(cls.ARCHIVE_ZIP)) configurations = [configuration] if configuration.is_complete( ) else ConfigurationContextTest.CONFIGURATIONS with model.upload_context: current = time.time() old = current - cls.THREE_WEEKS for complete_configuration in configurations: if complete_configuration != configuration: continue timestamp_to_use = current if (complete_configuration.platform == 'Mac' and complete_configuration.version <= Configuration.version_to_integer('10.13')) \ or (complete_configuration.platform == 'iOS' and complete_configuration.version <= Configuration.version_to_integer('11')): timestamp_to_use = old cls.iterate_all_commits( model, lambda commits: model.archive_context.register( archive, complete_configuration, commits, suite=suite, timestamp=timestamp_to_use))
def process_test_results(self, configuration, commits, suite, test_results, timestamp=None): timestamp = timestamp or time.time() if not self._async_processing: return self.synchronously_process_test_results(configuration, commits, suite, test_results=test_results, timestamp=timestamp) for branch in self.commit_context.branch_keys_for_commits(commits): hash_key = hash(configuration) ^ hash(branch) ^ hash(self.commit_context.uuid_for_commits(commits)) ^ hash( suite) self.redis.set( f'{self.QUEUE_NAME}:{hash_key}', json.dumps(dict(started_processing=0, attempts=0)), ex=self.PROCESS_TIMEOUT, ) self.redis.set( f'data_for_{self.QUEUE_NAME}:{hash_key}', json.dumps(dict( configuration=Configuration.Encoder().default(configuration), suite=suite, commits=Commit.Encoder().default(commits), timestamp=timestamp, test_results=test_results, )), ex=self.PROCESS_TIMEOUT, ) return {key: dict(status='Queued') for key in list(self._process_upload_callbacks[suite].keys()) + list(self._process_upload_callbacks[None].keys())}
def test_suite_list(self, redis=StrictRedis, cassandra=CassandraContext): self.init_database(redis=redis, cassandra=cassandra) MockModelFactory.add_mock_results(self.model) for suites in self.model.upload_context.find_suites(configurations=[Configuration()], recent=True).values(): self.assertEqual(suites, ['layout-tests']) MockModelFactory.add_mock_results(self.model, suite='api_tests') for suites in self.model.upload_context.find_suites(configurations=[Configuration()], recent=True).values(): self.assertEqual(suites, ['api_tests', 'layout-tests']) MockModelFactory.add_mock_results(self.model, configuration=Configuration(is_simulator=True), suite='webkitpy') for suites in self.model.upload_context.find_suites(configurations=[Configuration(is_simulator=True)], recent=True).values(): self.assertEqual(suites, ['api_tests', 'layout-tests', 'webkitpy']) for suites in self.model.upload_context.find_suites(configurations=[Configuration(is_simulator=False)], recent=True).values(): self.assertEqual(suites, ['api_tests', 'layout-tests'])
def download(self): AssertRequest.is_type(['GET']) with self.upload_context: uploads = self._find_uploads_for_query() response = [] for config, suite_results in uploads.items(): for suite, results in suite_results.items(): for result in results: config.sdk = result.get('sdk') response.append( dict( configuration=Configuration.Encoder().default( config), suite=suite, commits=[ commit.Encoder().default(commit) for commit in result['commits'] ], timestamp=result['timestamp'], test_results=result['test_results'], )) return jsonify(response)
def add_mock_results(cls, model, configuration=Configuration(), suite='layout-tests', test_results=None): if test_results is None: test_results = cls.layout_test_results() configurations = [configuration] if configuration.is_complete( ) else ConfigurationContextTest.CONFIGURATIONS with model.upload_context: current = time.time() old = current - 60 * 60 * 24 * 21 for complete_configuration in configurations: if complete_configuration != configuration: continue timestamp_to_use = current if (complete_configuration.platform == 'Mac' and complete_configuration.version <= Configuration.version_to_integer('10.13')) \ or (complete_configuration.platform == 'iOS' and complete_configuration.version <= Configuration.version_to_integer('11')): timestamp_to_use = old cls.iterate_all_commits( model, lambda commits: model.upload_context. upload_test_results(complete_configuration, commits, suite=suite, test_results=test_results, timestamp=timestamp_to_use))
def test_invalid_configuration(self, redis=StrictRedis, cassandra=CassandraContext): self.init_database(redis=redis, cassandra=cassandra) with self.assertRaises(TypeError): self.database.register_configuration('invalid object') with self.assertRaises(TypeError): self.database.register_configuration(Configuration(platform='iOS'))
def process(self): AssertRequest.is_type(['POST']) with self.upload_context: uploads = self._find_uploads_for_query() if not uploads: abort(404, description='No uploads matching the specified criteria') response = [] for config, suite_results in uploads.items(): for suite, results in suite_results.items(): for result in results: config.sdk = result.get('sdk') processing_results = self.upload_context.process_test_results( configuration=config, commits=result['commits'], suite=suite, test_results=result['test_results'], timestamp=result['timestamp'], ) response.append( dict( configuration=Configuration.Encoder().default( config), suite=suite, commits=Commit.Encoder().default( result['commits']), timestamp=result['timestamp'], processing=processing_results, )) return jsonify(response)
def process_results(self, model, configuration=Configuration(), suite='layout-tests'): configurations = [configuration] if configuration.is_complete( ) else ConfigurationContextTest.CONFIGURATIONS with model.upload_context: for complete_configuration in configurations: if complete_configuration != configuration: continue for branch in (None, 'safari-606-branch'): results_dict = model.upload_context.find_test_results( configurations=[complete_configuration], suite=suite, branch=branch, recent=False, ) for config, results in results_dict.items(): for result in results: model.upload_context.process_test_results( configuration=config, commits=result['commits'], suite=suite, test_results=result['test_results'], timestamp=result['timestamp'], )
def to_configuration(self): return Configuration(platform=self.platform, version=self.version, is_simulator=self.is_simulator, architecture=self.architecture, sdk=getattr(self, 'sdk', None) or None, **json.loads(self.attributes))
def test_sdk_differentiation(self, redis=StrictRedis, cassandra=CassandraContext): self.init_database(redis=redis, cassandra=cassandra) MockModelFactory.add_mock_results(self.model) configuration_to_search = Configuration(platform='iOS', version='12.0.0', is_simulator=True, style='Asan') results = self.model.upload_context.find_test_results(configurations=[configuration_to_search], suite='layout-tests', recent=False) self.assertEqual(1, len(results)) MockModelFactory.add_mock_results(self.model, configuration=Configuration( platform='iOS', version='12.0.0', sdk='16A405', is_simulator=True, architecture='x86_64', style='Asan', )) results = self.model.upload_context.find_test_results(configurations=[configuration_to_search], suite='layout-tests', recent=False) self.assertEqual(2, len(results)) results = self.model.upload_context.find_test_results(configurations=[Configuration(platform='iOS', sdk='16A405')], suite='layout-tests', recent=False) self.assertEqual(1, len(results))
def test_all_successful(self, redis=StrictRedis, cassandra=CassandraContext): self.init_database(redis=redis, cassandra=cassandra) results = self.model.suite_context.find_by_start_time( configurations=[ Configuration(platform='Mac', style='Release', flavor='wk1') ], suite='layout-tests', begin=MockSVNRepository.webkit().commit_for_id(236542), end=MockSVNRepository.webkit().commit_for_id(236542), ) self.assertEqual( next(iter(results.values()))[0]['stats'], dict( tests_run=4, tests_skipped=0, tests_failed=0, tests_timedout=0, tests_crashed=0, tests_unexpected_failed=0, tests_unexpected_timedout=0, tests_unexpected_crashed=0, ), )
def select_from_table_with_configurations(self, table_name, configurations=None, branch=None, recent=True, limit=100, **kwargs): if not isinstance(configurations, Iterable): raise TypeError('Expected configurations to be iterable') if not configurations: configurations.append(Configuration()) with self: complete_configurations = set() for config in configurations: if not isinstance(config, Configuration): raise TypeError( f'Expected type {Configuration}, got {type(config)}') if config.is_complete(): complete_configurations.add(config) elif recent: [ complete_configurations.add(element) for element in self.search_for_recent_configuration( config, branch=branch) ] else: [ complete_configurations.add(element) for element in self.search_for_configuration(config, branch=branch) ] results = {} for configuration in complete_configurations: attributes_dict = OrderedDict() for member in Configuration.OPTIONAL_MEMBERS: if getattr(configuration, member) is not None: attributes_dict[member] = getattr( configuration, member) rows = self.cassandra.select_from_table( table_name, platform=configuration.platform, is_simulator=configuration.is_simulator, version=configuration.version, architecture=configuration.architecture, attributes=json.dumps(attributes_dict), limit=limit, branch=branch or CommitContext.DEFAULT_BRANCH_KEY, **kwargs) if len(rows) == 0: continue for row in rows: full_config = row.to_configuration() if not results.get(full_config): results[full_config] = [] results[full_config].append(row) return results
def _do_job_for_key(self, key, attempts=1): job_complete = False try: raw_data = self.redis.get(f'data_for_{key}') if raw_data: data = json.loads(raw_data) self.synchronously_process_test_results( configuration=Configuration.from_json( data['configuration']), commits=[ Commit.from_json(commit_json) for commit_json in data['commits'] ], suite=data['suite'], timestamp=data['timestamp'], test_results=data['test_results'], ) job_complete = True finally: if job_complete or attempts >= self.MAX_ATTEMPTS: self.redis.delete(key) self.redis.delete(f'data_for_{key}') else: self.redis.set( key, json.dumps(dict(started_processing=0, attempts=attempts)), ex=self.PROCESS_TIMEOUT, )
def test_no_test_runs(self, redis=StrictRedis, cassandra=CassandraContext): self.init_database(redis=redis, cassandra=cassandra) results = self.model.failure_context.failures_by_commit( configurations=[Configuration(platform='Mac', style='Release', flavor='wk1')], suite='layout-tests', recent=True, end=0, ) self.assertEqual(results, None)
def test_configuration_by_model(self, redis=StrictRedis, cassandra=CassandraContext): self.init_database(redis=redis, cassandra=cassandra) self.register_configurations() configuration_to_search_for = Configuration(model='iPhone 8') matching_configurations = self.database.search_for_configuration(configuration_to_search_for) self.assertEqual(4, len(matching_configurations)) for config in matching_configurations: self.assertEqual(configuration_to_search_for, config)
def search_for_recent_configuration(self, configuration=Configuration(), branch=None): if not isinstance(configuration, Configuration): raise TypeError( f'Expected type {Configuration}, got {type(configuration)}') configurations = [] for key in self.redis.scan_iter( self._convert_to_redis_key( configuration, branch or CommitContext.DEFAULT_BRANCH_KEY)): candidate = Configuration.from_json( self.redis.get(key.decode('utf-8')).decode('utf-8')) if candidate == configuration: configurations.append(candidate) return configurations
def test_result_retrieval(self, redis=StrictRedis, cassandra=CassandraContext): self.init_database(redis=redis, cassandra=cassandra) MockModelFactory.add_mock_results(self.model) results = self.model.upload_context.find_test_results( configurations=[Configuration(platform='Mac')], suite='layout-tests', recent=True) self.assertEqual(6, len(results)) for config, values in results.items(): self.assertEqual(config, Configuration(platform='Mac')) self.assertEqual(5, len(values)) for value in values: self.assertEqual(value['test_results'], MockModelFactory.layout_test_results())
def test_recent_configurations_constrained(self, redis=StrictRedis, cassandra=CassandraContext): self.init_database(redis=redis, cassandra=cassandra) self.register_configurations() configuration_to_search_for = Configuration(architecture='arm64', style='Release') matching_configurations = self.database.search_for_recent_configuration(configuration_to_search_for) self.assertEqual(1, len(matching_configurations)) self.assertEqual(matching_configurations[0], configuration_to_search_for)
def find_run_results(self, suite=None, configurations=None, recent=None, branch=None, begin=None, end=None, begin_query_time=None, end_query_time=None, limit=None, **kwargs): AssertRequest.is_type(['GET']) AssertRequest.query_kwargs_empty(**kwargs) recent = boolean_query(*recent)[0] if recent else True with self.suite_context: if not suite: abort(400, description='No suite specified') query_dict = dict( suite=suite, configurations=configurations, recent=recent, branch=branch[0], begin=begin, end=end, begin_query_time=begin_query_time, end_query_time=end_query_time, limit=limit, ) specified_commits = sum( [1 if element else 0 for element in [begin, end]]) specified_timestamps = sum([ 1 if element else 0 for element in [begin_query_time, end_query_time] ]) if specified_commits >= specified_timestamps: find_function = self.suite_context.find_by_commit def sort_function(result): return result['uuid'] else: find_function = self.suite_context.find_by_start_time def sort_function(result): return result['start_time'] response = [] for config, results in find_function(**query_dict).items(): response.append( dict( configuration=Configuration.Encoder().default(config), results=sorted(results, key=sort_function), )) return jsonify(response)
def _convert_to_redis_key(cls, configuration, branch): return 'configs_with_branch:{}:{}:{}:{}:{}:{}:{}:{}:{}'.format( configuration.platform or '*', '*' if configuration.is_simulator is None else (1 if configuration.is_simulator else 0), '*' if configuration.version is None else Configuration.integer_to_version(configuration.version), configuration.version_name or '*', configuration.architecture or '*', configuration.model or '*', configuration.style or '*', configuration.flavor or '*', branch, )
def test_recent_configurations_constrained_by_version(self, redis=StrictRedis, cassandra=CassandraContext): self.init_database(redis=redis, cassandra=cassandra) self.register_configurations() configuration_to_search_for = Configuration(version='12', is_simulator=True) matching_configurations = self.database.search_for_recent_configuration(configuration_to_search_for) self.assertEqual(3, len(matching_configurations)) for config in matching_configurations: self.assertEqual(configuration_to_search_for, config)
def test_configuration_by_platform(self, redis=StrictRedis, cassandra=CassandraContext): self.init_database(redis=redis, cassandra=cassandra) self.register_configurations() configuration_to_search_for = Configuration(platform='Mac', style='Debug') matching_configurations = self.database.search_for_configuration(configuration_to_search_for) self.assertEqual(4, len(matching_configurations)) for config in matching_configurations: self.assertEqual(configuration_to_search_for, config)
def test_configuration_by_architecture(self, redis=StrictRedis, cassandra=CassandraContext): self.init_database(redis=redis, cassandra=cassandra) self.register_configurations() configuration_to_search_for = Configuration(architecture='x86_64', style='Release') matching_configurations = self.database.search_for_configuration(configuration_to_search_for) self.assertEqual(6, len(matching_configurations)) for config in matching_configurations: self.assertEqual(configuration_to_search_for, config)
def to_configuration(self): return Configuration( platform=self.platform, version=self.version, version_name=self.version_name or None, sdk=getattr(self, 'sdk', None) or None, is_simulator=self.is_simulator, architecture=self.architecture, model=self.model or None, style=self.style or None, flavor=self.flavor or None, )
def test_unexpected_failures_collapsed(self, redis=StrictRedis, cassandra=CassandraContext): self.init_database(redis=redis, cassandra=cassandra) results = self.model.failure_context.failures_by_commit( configurations=[Configuration(platform='Mac', style='Release', flavor='wk1')], suite='layout-tests', recent=True, ) self.assertEqual(len(results), 1) self.assertEqual(results, set(['fast/encoding/css-cached-bom.html']))
def test_file_list(self, redis=StrictRedis, cassandra=CassandraContext): self.init_database(redis=redis, cassandra=cassandra) files = self.model.archive_context.file( configurations=[Configuration(platform='Mac', style='Release', flavor='wk1')], begin=1601660000, end=1601660000, suite='layout-tests', ) self.assertEqual(len(next(iter(files.values()))), 1) self.assertEqual(next(iter(files.values()))[0]['uuid'], 160166000000) self.assertEqual(next(iter(files.values()))[0]['file'], ['file.txt', 'index.html'])
def test_find_archive(self, redis=StrictRedis, cassandra=CassandraContext): self.init_database(redis=redis, cassandra=cassandra) archives = self.model.archive_context.find_archive( configurations=[Configuration(platform='Mac', style='Release', flavor='wk1')], begin=1601660000, end=1601660000, suite='layout-tests', ) self.assertEqual(len(next(iter(archives.values()))), 1) self.assertEqual(next(iter(archives.values()))[0]['uuid'], 160166000000) self.assertEqual(next(iter(archives.values()))[0]['archive'].getvalue(), base64.b64decode(MockModelFactory.ARCHIVE_ZIP))
def test_file_list(self, redis=StrictRedis, cassandra=CassandraContext): self.init_database(redis=redis, cassandra=cassandra) files = self.model.archive_context.file( configurations=[Configuration(platform='Mac', style='Release', flavor='wk1')], begin=MockSVNRepository.webkit().commit_for_id(236542), end=MockSVNRepository.webkit().commit_for_id(236542), suite='layout-tests', ) self.assertEqual(len(next(iter(files.values()))), 1) self.assertEqual(next(iter(files.values()))[0]['uuid'], 153804910800) self.assertEqual(next(iter(files.values()))[0]['file'], ['file.txt', 'index.html'])
def test_no_failures(self, redis=StrictRedis, cassandra=CassandraContext): cassandra.drop_keyspace(keyspace=self.KEYSPACE) self.model = MockModelFactory.create(redis=redis(), cassandra=cassandra(keyspace=self.KEYSPACE, create_keyspace=True)) MockModelFactory.add_mock_results(self.model) MockModelFactory.process_results(self.model) results = self.model.failure_context.failures_by_commit( configurations=[Configuration(platform='Mac', style='Release', flavor='wk1')], suite='layout-tests', recent=True, collapsed=False, unexpected=False, ) self.assertEqual(len(results), 0)