def suites(self, configurations=None, recent=None, suite=None, branch=None, **kwargs): AssertRequest.is_type(['GET']) AssertRequest.query_kwargs_empty(**kwargs) with self.upload_context: suites_by_config = self.upload_context.find_suites( configurations=configurations, recent=boolean_query(*recent)[0] if recent else True, branch=branch[0] if branch else None, ) result = [] for config, candidate_suites in suites_by_config.items(): suites_for_config = [ s for s in candidate_suites if not suite or s in suite ] if suites_for_config: result.append([config, suites_for_config]) if not result: abort(404, description='No suites matching the specified criteria') return jsonify(Configuration.Encoder().default(result))
def library(self, path): AssertRequest.no_query() AssertRequest.is_type() path_split = os.path.split(path) return send_from_directory( os.path.join(os.path.abspath(os.path.dirname(__file__)), 'static/library', *path_split[:-1]), path_split[-1])
def _suites_for_investigation(self, suite=None, configurations=None, recent=None, branch=None, begin=None, end=None, begin_query_time=None, end_query_time=None, unexpected=None, limit=None, **kwargs): AssertRequest.is_type(['GET']) AssertRequest.query_kwargs_empty(**kwargs) with self.upload_controller.upload_context: suites_by_configuration = self.upload_controller.upload_context.find_suites( configurations=configurations, recent=boolean_query(*recent)[0] if recent else True) candidate_suites = set() for suites_for_config in suites_by_configuration.values(): for s in suites_for_config: candidate_suites.add(s) return sorted( [s for s in candidate_suites if not suite or s in suite])
def download(self): AssertRequest.is_type(['GET']) with self.upload_context: uploads = self._find_uploads_for_query() response = [] for config, suite_results in uploads.items(): for suite, results in suite_results.items(): for result in results: config.sdk = result.get('sdk') response.append( dict( configuration=Configuration.Encoder().default( config), suite=suite, commits=[ commit.Encoder().default(commit) for commit in result['commits'] ], timestamp=result['timestamp'], test_results=result['test_results'], )) return jsonify(response)
def process(self): AssertRequest.is_type(['POST']) with self.upload_context: uploads = self._find_uploads_for_query() if not uploads: abort(404, description='No uploads matching the specified criteria') response = [] for config, suite_results in uploads.items(): for suite, results in suite_results.items(): for result in results: config.sdk = result.get('sdk') processing_results = self.upload_context.process_test_results( configuration=config, commits=result['commits'], suite=suite, test_results=result['test_results'], timestamp=result['timestamp'], ) response.append(dict( configuration=Configuration.Encoder().default(config), suite=suite, commits=Commit.Encoder().default(result['commits']), timestamp=result['timestamp'], processing=processing_results, )) return jsonify(response)
def _suites_for_search(self, suite=None, configurations=None, recent=None, branch=None, begin=None, end=None, begin_query_time=None, end_query_time=None, test=None, limit=None, **kwargs): AssertRequest.is_type(['GET']) AssertRequest.query_kwargs_empty(**kwargs) if test and (not suite or len(suite) != len(test)): abort(400, description='Each test must be paired with a suite') with self.upload_controller.upload_context: suites_by_configuration = self.upload_controller.upload_context.find_suites( configurations=configurations, recent=boolean_query(*recent)[0] if recent else True) candidate_suites = set() for suites_for_config in suites_by_configuration.values(): for s in suites_for_config: candidate_suites.add(s) return sorted([ s for s in candidate_suites if test or not suite or s in suite ])
def representations(self): AssertRequest.is_type() AssertRequest.no_query() return jsonify({ key: repository.representations() for key, repository in self.commit_context.repositories.items() })
def find(self): AssertRequest.is_type() result = self._find(**request.args.to_dict(flat=False)) if not result: abort( 404, description='No commits found matching the specified criteria') return jsonify(Commit.Encoder().default(result))
def find_run_results(self, suite=None, configurations=None, recent=None, branch=None, begin=None, end=None, begin_query_time=None, end_query_time=None, limit=None, **kwargs): AssertRequest.is_type(['GET']) AssertRequest.query_kwargs_empty(**kwargs) recent = boolean_query(*recent)[0] if recent else True with self.suite_context: if not suite: abort(400, description='No suite specified') query_dict = dict( suite=suite, configurations=configurations, recent=recent, branch=branch[0], begin=begin, end=end, begin_query_time=begin_query_time, end_query_time=end_query_time, limit=limit, ) specified_commits = sum( [1 if element else 0 for element in [begin, end]]) specified_timestamps = sum([ 1 if element else 0 for element in [begin_query_time, end_query_time] ]) if specified_commits >= specified_timestamps: find_function = self.suite_context.find_by_commit def sort_function(result): return result['uuid'] else: find_function = self.suite_context.find_by_start_time def sort_function(result): return result['start_time'] response = [] for config, results in find_function(**query_dict).items(): response.append( dict( configuration=Configuration.Encoder().default(config), results=sorted(results, key=sort_function), )) return jsonify(response)
def register(self, commit=None): is_endpoint = not bool(commit) if is_endpoint: AssertRequest.is_type(['POST']) AssertRequest.no_query() if is_endpoint: try: commit = request.form or json.loads(request.get_data()) if 'api_key' in commit: del commit['api_key'] except ValueError: abort(400, description='Expected uploaded data to be json') try: self.commit_context.register_commit(Commit.from_json(commit)) if is_endpoint: return jsonify({'status': 'ok'}) return Commit.from_json(commit) except ValueError: pass required_args = ['repository_id', 'id'] optional_args = ['branch'] for arg in required_args: if arg not in commit: abort(400, description=f"'{arg}' required to define commit") for arg in commit.keys(): if arg in required_args + optional_args: continue if arg in ['timestamp', 'order', 'committer', 'message']: abort( 400, description= 'Not enough arguments provided to define a commit, but too many to search for a commit' ) abort(400, description=f"'{arg}' is not valid for defining commits") try: commit = self.commit_context.register_commit_with_repo_and_id( repository_id=commit.get('repository_id'), branch=commit.get('branch'), commit_id=commit.get('id'), ) except (RuntimeError, SCMException) as error: abort(404, description=str(error)) if is_endpoint: return jsonify({'status': 'ok'}) return commit
def list_tests(self, suite=None, test=None, limit=None, **kwargs): AssertRequest.is_type(['GET']) AssertRequest.query_kwargs_empty(**kwargs) if not suite: abort(400, description='No suite specified') with self.test_context: matching_tests = set() for t in test or [None]: matching_tests.update( self.test_context.names(suite=suite, test=t, limit=limit - len(matching_tests))) return jsonify(sorted(matching_tests))
def upload(self): if request.method == 'GET': return self.download() AssertRequest.is_type(['POST']) AssertRequest.no_query() with self.upload_context: try: data = request.form or json.loads(request.get_data()) except ValueError: abort(400, description='Expected uploaded data to be json') try: configuration = Configuration.from_json( data.get('configuration', {})) except (ValueError, TypeError): abort(400, description='Invalid configuration') suite = data.get('suite') if not suite: abort(400, description='No test suite specified') commits = [ self.commit_controller.register(commit=commit, fast=True) for commit in data.get('commits', []) ] test_results = data.get('test_results', {}) if not test_results: abort(400, description='No test results specified') timestamp = data.get('timestamp', time.time()) version = data.get('version', 0) try: self.upload_context.upload_test_results(configuration, commits, suite, test_results, timestamp, version=version) except (TypeError, ValueError) as error: abort(400, description=str(error)) processing_results = self.upload_context.process_test_results( configuration, commits, suite, test_results, timestamp) return jsonify(dict(status='ok', processing=processing_results))
def _single_commit(self, limit=None, **kwargs): AssertRequest.is_type() AssertRequest.query_kwargs_empty(limit=limit) with self.commit_context: commits = self.commit_controller._find(**kwargs) if not commits: abort(404, description= 'No commits found matching the specified criteria') if len(commits) > 1: abort( 404, description= f'{len(commits)} commits found matching the specified criteria' ) return commits[0]
def branches(self, repository_id=None, branch=None, limit=None, **kwargs): AssertRequest.is_type() AssertRequest.query_kwargs_empty(**kwargs) result = defaultdict(list) with self.commit_context: for repository in repository_id or self.commit_context.repositories.keys( ): limit_for_repo = limit for b in branch or [None]: if not limit_for_repo: continue matching_branches = self.commit_context.branches( repository, branch=b, limit=limit_for_repo) if not matching_branches: continue limit_for_repo -= len(matching_branches) result[repository] += matching_branches return jsonify(result)
def previous(self, limit=None, **kwargs): AssertRequest.is_type() AssertRequest.query_kwargs_empty(limit=limit) with self.commit_context: commits = self._find(**kwargs) if not commits: abort(404, description= 'No commits found matching the specified criteria') if len(commits) > 1: abort( 404, description= f'{len(commits)} commits found matching the specified criteria' ) commit = self.commit_context.previous_commit(commits[0]) return jsonify( Commit.Encoder().default([commit] if commit else []))
def siblings(self, limit=None, **kwargs): AssertRequest.is_type() AssertRequest.query_kwargs_empty(limit=limit) with self.commit_context: commits = self._find(**kwargs) if not commits: abort(404, description= 'No commits found matching the specified criteria') if len(commits) > 1: abort( 404, description= f'{len(commits)} commits found matching the specified criteria' ) repositories = sorted(self.commit_context.repositories.keys()) repositories.remove(commits[0].repository_id) return jsonify(Commit.Encoder().default( self.commit_context.sibling_commits(commits[0], repositories)))
def urls_for_queue(self, suite=None, branch=None, configurations=None, recent=None, limit=None, **kwargs): AssertRequest.is_type(['GET']) AssertRequest.query_kwargs_empty(**kwargs) is_recent = True if recent: is_recent = boolean_query(*recent)[0] with self.ci_context, self.upload_context: suites = self._suites_for_query_arguments( suite=suite, configurations=configurations, is_recent=is_recent) if not branch: branch = [None] results = [] for suite in suites: for config, url in self.ci_context.find_urls_by_queue( configurations=configurations, recent=is_recent, branch=branch[0], suite=suite, limit=limit, ).items(): configuration_dict = Configuration.Encoder().default( config) configuration_dict['suite'] = suite results.append( dict(configuration=configuration_dict, url=url)) return results
def upload(self): AssertRequest.is_type(['POST']) AssertRequest.no_query() with self.archive_context: if 'file' not in request.files: abort(400, description='No archive provided') archive = io.BytesIO(request.files['file'].read()) try: data = request.form or json.loads(request.get_data()) except ValueError: abort(400, description='Expected meta-data to be json') try: configuration = Configuration.from_json(data.get('configuration', {})) except (ValueError, TypeError) as e: abort(400, description=f'Invalid configuration, error: {e}') suite = data.get('suite') if not suite: abort(400, description='No test suite specified') try: commits = [self.commit_controller.register(commit=commit) for commit in json.loads(data.get('commits', '[]'))] except ValueError: abort(400, description='Expected commit meta-data to be json') if not commits: abort(400, description='No commits provided') timestamp = data.get('timestamp', time.time()) try: self.archive_context.register(archive, configuration, commits, suite, timestamp) except (TypeError, ValueError) as error: abort(400, description=str(error)) return jsonify(dict(status='ok'))
def download( self, suite=None, configurations=None, recent=None, branch=None, begin=None, end=None, begin_query_time=None, end_query_time=None, **kwargs ): AssertRequest.is_type(['GET']) AssertRequest.query_kwargs_empty(**kwargs) recent = boolean_query(*recent)[0] if recent else True if not suite: suites = set() for config_suites in self.upload_context.find_suites(configurations=configurations, recent=recent).values(): [suites.add(suite) for suite in config_suites] else: suites = set(suite) result = None filename = None digest = None with self.archive_context, self.upload_context: for suite in suites: for configuration, archives in self.archive_context.find_archive( configurations=configurations, suite=suite, branch=branch[0], begin=begin, end=end, recent=recent, limit=2, begin_query_time=begin_query_time, end_query_time=end_query_time, ).items(): for archive in archives: if archive.get('archive') and archive.get('digest'): if digest and digest != archive.get('digest'): abort(400, description='Multiple archives matching the specified criteria') result = archive.get('archive') filename = f'{configuration}@{archive["uuid"]}'.replace(' ', '_').replace('.', '-') digest = archive.get('digest') if not result: abort(404, description='No archives matching the specified criteria') return send_file(result, attachment_filename=f'{filename}.zip', as_attachment=True)
def extract( self, path=None, format=None, suite=None, configurations=None, recent=None, branch=None, begin=None, end=None, begin_query_time=None, end_query_time=None, limit=None, **kwargs ): AssertRequest.is_type(['GET']) AssertRequest.query_kwargs_empty(**kwargs) recent = boolean_query(*recent)[0] if recent else True if not suite: suites = set() for config_suites in self.upload_context.find_suites(configurations=configurations, recent=recent).values(): [suites.add(suite) for suite in config_suites] else: suites = set(suite) result = None with self.archive_context, self.upload_context: for suite in suites: for files in self.archive_context.file( path=path, configurations=configurations, suite=suite, branch=branch[0], begin=begin, end=end, recent=recent, limit=2, begin_query_time=begin_query_time, end_query_time=end_query_time, ).values(): for file in files: candidate = file.get('file') if not candidate: continue if not result: if isinstance(candidate, list): result = set(candidate) else: result = candidate continue if isinstance(candidate, list) and isinstance(result, set): result |= set(candidate) continue if result == candidate: continue abort(403, 'Multiple archives match with different content') if not result: abort(404, f"No archive content{' at ' + path if path else ''}") if isinstance(result, set): return self.list(path=path, values=sorted(result)) mimetype = self.FORMAT.get(format) if not mimetype and '.' in path: mimetype = self.FORMAT.get(path.split('.')[-1]) mimetype = mimetype or 'text/plain' # A bit of a hack to add the right query arguments into results.html # Ideally, this should be done with changes to results.html, but we # have legacy results to handle. if mimetype == 'text/html': query = query_as_string() result = result.replace( "href=\"' + testPrefix + suffix + '\"".encode('utf-8'), f"href=\"' + testPrefix + suffix + '{query}' + '\"".encode('utf-8'), ) for include_to_strip in ['js/status-bubble.js', 'code-review.js?version=48']: result = result.replace(f'<script src="{include_to_strip}"></script>'.encode('utf-8'), ''.encode('utf-8')) if (query): result = result.replace("src += '?format=txt'".encode('utf-8'), "src += '&format=txt'".encode('utf-8')) return Response(result, mimetype=mimetype or 'text/plain')
def repositories(self): AssertRequest.is_type() AssertRequest.no_query() return jsonify(sorted(self.commit_context.repositories.keys()))
def register(self, commit=None, fast=True): is_endpoint = not bool(commit) if is_endpoint: AssertRequest.is_type(['POST']) AssertRequest.no_query() if is_endpoint: try: commit = request.form or json.loads(request.get_data()) if 'api_key' in commit: del commit['api_key'] except ValueError: abort(400, description='Expected uploaded data to be json') try: candidate = ScmCommit.from_json(commit) # Commit needs to be sufficiently defined if candidate.repository_id and candidate.branch and candidate.timestamp and ( candidate.revision or candidate.hash): self.commit_context.register_commit(candidate) if is_endpoint: return jsonify({'status': 'ok'}) return candidate except ValueError: pass required_args = ['repository_id'] for arg in required_args: if arg not in commit: abort(400, description=f"'{arg}' required to define commit") has_ref = False one_of_args = ['id', 'ref', 'hash', 'revision', 'identifier'] for arg in one_of_args: if arg in commit: if has_ref: abort(400, description='Multiple commit references specified') has_ref = True if not has_ref: abort(400, description='No commit reference specified') for arg in commit.keys(): if arg in required_args or arg in one_of_args: continue if arg in ['branch', 'timestamp', 'order', 'committer', 'message']: abort( 400, description= 'Not enough arguments provided to define a commit, but too many to search for a commit' ) abort(400, description=f"'{arg}' is not valid for defining commits") try: commit = self.commit_context.register_partial_commit( repository_id=commit.get('repository_id'), ref=commit.get('id') or commit.get('ref'), hash=commit.get('hash'), revision=commit.get('revision'), identifier=commit.get('identifier'), fast=fast, ) except (RuntimeError, ScmBase.Exception) as error: abort(404, description=str(error)) if is_endpoint: return jsonify({'status': 'ok'}) return Commit( repository_id=commit.repository_id, id=commit.revision or commit.hash, branch=commit.branch, timestamp=commit.timestamp, order=commit.order, committer=commit.author.email if commit.author else None, message=commit.message, )
def failures(self, suite=None, configurations=None, recent=None, branch=None, begin=None, end=None, begin_query_time=None, end_query_time=None, unexpected=None, collapsed=None, limit=None, **kwargs): AssertRequest.is_type(['GET']) AssertRequest.query_kwargs_empty(**kwargs) recent = boolean_query(*recent)[0] if recent else True unexpected = boolean_query(*unexpected)[0] if unexpected else True collapsed = boolean_query(*collapsed)[0] if collapsed else True if not suite: abort(400, description='No suite specified') with self.failure_context: query_dict = dict( suite=suite, configurations=configurations, recent=recent, branch=branch[0], begin=begin, end=end, begin_query_time=begin_query_time, end_query_time=end_query_time, limit=limit, unexpected=unexpected, collapsed=collapsed, ) num_specified_commits = sum( [1 if element else 0 for element in [begin, end]]) num_specified_timestamps = sum([ 1 if element else 0 for element in [begin_query_time, end_query_time] ]) if num_specified_commits >= num_specified_timestamps: find_function = self.failure_context.failures_by_commit def sort_function(result): return result['uuid'] else: find_function = self.failure_context.failures_by_start_time def sort_function(result): return result['start_time'] failures = find_function(**query_dict) if failures is None: abort( 404, description='No test runs found with the specified criteria' ) if collapsed: return jsonify(sorted(set(failures))) response = [] for config, results in failures.items(): response.append( dict( configuration=Configuration.Encoder().default(config), results=sorted(results, key=sort_function), )) return jsonify(response)
def urls_for_builds(self, suite=None, configurations=None, recent=None, branch=None, begin=None, end=None, begin_query_time=None, end_query_time=None, limit=None, **kwargs): AssertRequest.is_type(['GET']) AssertRequest.query_kwargs_empty(**kwargs) is_recent = True if recent: is_recent = boolean_query(*recent)[0] with self.ci_context, self.upload_context: suites = self._suites_for_query_arguments( suite=suite, configurations=configurations, is_recent=is_recent) if not branch: branch = [None] query_dict = dict( configurations=configurations, recent=is_recent, branch=branch[0], begin=begin, end=end, begin_query_time=begin_query_time, end_query_time=end_query_time, limit=limit, ) num_uuid_query_args = sum( [1 if element else 0 for element in [begin, end]]) num_timestamp_query_args = sum([ 1 if element else 0 for element in [begin_query_time, end_query_time] ]) if num_uuid_query_args >= num_timestamp_query_args: find_function = self.ci_context.find_urls_by_commit def sort_function(result): return result['uuid'] else: find_function = self.ci_context.find_urls_by_start_time def sort_function(result): return result['start_time'] results = [] for suite in suites: for config, urls in find_function(suite=suite, **query_dict).items(): configuration_dict = Configuration.Encoder().default( config) configuration_dict['suite'] = suite results.append( dict(configuration=configuration_dict, urls=sorted(urls, key=sort_function))) return results