def build_list(job_dir, before): ''' Given a job dir, give a (partial) list of recent build finished.jsons. Args: job_dir: the GCS path holding the jobs Returns: a list of [(build, loc, started, finished)]. build is a string like "123", loc is the job directory and build, started/finished are either None or a dict of the finished.json, and a dict of {build: [issues...]} of xrefs. ''' # /directory/ folders have a series of .txt files pointing at the correct location, # as a sort of fake symlink. indirect = '/directory/' in job_dir builds = get_build_numbers(job_dir, before, indirect) if indirect: # follow the indirect links build_symlinks = [ (build, gcs_async.read('%s%s.txt' % (job_dir, build))) for build in builds ] build_futures = [] for build, sym_fut in build_symlinks: redir = sym_fut.get_result() if redir and redir.startswith('gs://'): redir = redir[4:].strip() build_futures.append( (build, redir, gcs_async.read('%s/started.json' % redir), gcs_async.read('%s/finished.json' % redir))) else: build_futures = [ (build, '%s%s' % (job_dir, build), gcs_async.read('%s%s/started.json' % (job_dir, build)), gcs_async.read('%s%s/finished.json' % (job_dir, build))) for build in builds ] refs = {} output = [] for build, loc, started_future, finished_future in build_futures: started, finished, metadata = normalize_metadata(started_future, finished_future) issues = [] if "repos" in metadata and metadata["repos"]: for repo in metadata['repos']: for ref in metadata['repos'][repo].split(','): x = ref.split(':', 1) if len(x) == 2 and x[0].isdigit(): issues.append({"number": int(x[0]), "title": "", "url": "https://github.com/%s/pull/%s" % (repo, x[0]) }) refs[loc] = issues output.append((str(build), loc, started, finished)) return output, refs
def pr_builds(path): """Return {job: [(build, {started.json}, {finished.json})]} for each job under gcs path.""" jobs_dirs_fut = gcs_async.listdirs(path) def base(path): return os.path.basename(os.path.dirname(path)) jobs_futures = [(job, gcs_async.listdirs(job)) for job in jobs_dirs_fut.get_result()] futures = [] for job, builds_fut in jobs_futures: for build in builds_fut.get_result(): futures.append([ base(job), base(build), gcs_async.read('/%sstarted.json' % build), gcs_async.read('/%sfinished.json' % build)]) futures.sort(key=lambda (job, build, s, f): (job, view_base.pad_numbers(build)), reverse=True) jobs = {} for job, build, started_fut, finished_fut in futures: started = started_fut.get_result() finished = finished_fut.get_result() if started is not None: started = json.loads(started) if finished is not None: finished = json.loads(finished) jobs.setdefault(job, []).append((build, started, finished)) return jobs
def build_details(build_dir): """ Collect information from a build directory. Args: build_dir: GCS path containing a build's results. Returns: started: value from started.json {'version': ..., 'timestamp': ...} finished: value from finished.json {'timestamp': ..., 'result': ...} results: {total: int, failed: [(name, duration, text)...], skipped: [name...], passed: [name...]} """ started_fut = gcs_async.read(build_dir + '/started.json') finished = gcs_async.read(build_dir + '/finished.json').get_result() started = started_fut.get_result() if finished and not started: started = 'null' if started and not finished: finished = 'null' elif not (started and finished): return started = json.loads(started) finished = json.loads(finished) junit_paths = [f.filename for f in view_base.gcs_ls('%s/artifacts' % build_dir) if re.match(r'junit_.*\.xml', os.path.basename(f.filename))] junit_futures = {f: gcs_async.read(f) for f in junit_paths} parser = JUnitParser() for path, future in junit_futures.iteritems(): parser.parse_xml(future.get_result(), path) return started, finished, parser.get_results()
def get_build_numbers(job_dir, before, indirect): try: if 'pr-logs' in job_dir and not indirect: raise ValueError('bad code path for PR build list') # If we have latest-build.txt, we can skip an expensive GCS ls call! if before: latest_build = int(before) - 1 else: latest_build = int(gcs_async.read(job_dir + 'latest-build.txt').get_result()) # latest-build.txt has the most recent finished build. There might # be newer builds that have started but not finished. Probe for them. suffix = '/started.json' if not indirect else '.txt' while gcs_async.read('%s%s%s' % (job_dir, latest_build + 1, suffix)).get_result(): latest_build += 1 return range(latest_build, max(0, latest_build - 40), -1) except (ValueError, TypeError): fstats = view_base.gcs_ls(job_dir) fstats.sort(key=lambda f: view_base.pad_numbers(f.filename), reverse=True) if indirect: # find numbered builds builds = [re.search(r'/(\d*)\.txt$', f.filename) for f in fstats if not f.is_dir] builds = [m.group(1) for m in builds if m] else: builds = [os.path.basename(os.path.dirname(f.filename)) for f in fstats if f.is_dir] if before and before in builds: builds = builds[builds.index(before) + 1:] return builds[:40]
def build_details(build_dir): """ Collect information from a build directory. Args: build_dir: GCS path containing a build's results. Returns: started: value from started.json {'version': ..., 'timestamp': ...} finished: value from finished.json {'timestamp': ..., 'result': ...} results: {total: int, failed: [(name, duration, text)...], skipped: [name...], passed: [name...]} """ started, finished, metadata = normalize_metadata( gcs_async.read(build_dir + '/started.json'), gcs_async.read(build_dir + '/finished.json') ) if started is None and finished is None: return started, finished, {}, None junit_paths = [f.filename for f in view_base.gcs_ls_recursive('%s/artifacts' % build_dir) if f.filename.endswith('.xml')] junit_futures = {f: gcs_async.read(f) for f in junit_paths} parser = JUnitParser() for path, future in junit_futures.iteritems(): parser.parse_xml(future.get_result(), path) return started, finished, metadata, parser.get_results()
def build_details(build_dir): """ Collect information from a build directory. Args: build_dir: GCS path containing a build's results. Returns: started: value from started.json {'version': ..., 'timestamp': ...} finished: value from finished.json {'timestamp': ..., 'result': ...} results: {total: int, failed: [(name, duration, text)...], skipped: [name...], passed: [name...]} """ started, finished = normalize_metadata( gcs_async.read(build_dir + '/started.json'), gcs_async.read(build_dir + '/finished.json') ) if started is None and finished is None: return started, finished, None junit_paths = [f.filename for f in view_base.gcs_ls_recursive('%s/artifacts' % build_dir) if f.filename.endswith('.xml')] junit_futures = {f: gcs_async.read(f) for f in junit_paths} parser = JUnitParser() for path, future in junit_futures.iteritems(): parser.parse_xml(future.get_result(), path) return started, finished, parser.get_results()
def get_build_numbers(job_dir, before, indirect): try: if '/pull/' in job_dir and not indirect: raise ValueError('bad code path for PR build list') # If we have latest-build.txt, we can skip an expensive GCS ls call! if before: latest_build = int(before) - 1 else: latest_build = int(gcs_async.read(job_dir + 'latest-build.txt').get_result()) # latest-build.txt has the most recent finished build. There might # be newer builds that have started but not finished. Probe for them. suffix = '/started.json' if not indirect else '.txt' while gcs_async.read('%s%s%s' % (job_dir, latest_build + 1, suffix)).get_result(): latest_build += 1 return range(latest_build, max(0, latest_build - 40), -1) except (ValueError, TypeError): fstats = view_base.gcs_ls(job_dir) fstats.sort(key=lambda f: view_base.pad_numbers(f.filename), reverse=True) if indirect: # find numbered builds builds = [re.search(r'/(\d*)\.txt$', f.filename) for f in fstats if not f.is_dir] builds = [m.group(1) for m in builds if m] else: builds = [os.path.basename(os.path.dirname(f.filename)) for f in fstats if f.is_dir] if before and before in builds: builds = builds[builds.index(before) + 1:] return builds[:40]
def pr_builds(path): """Return {job: [(build, {started.json}, {finished.json})]} for each job under gcs path.""" jobs_dirs_fut = gcs_async.listdirs(path) def base(path): return os.path.basename(os.path.dirname(path)) jobs_futures = [(job, gcs_async.listdirs(job)) for job in jobs_dirs_fut.get_result()] futures = [] for job, builds_fut in jobs_futures: for build in builds_fut.get_result(): futures.append([ base(job), base(build), gcs_async.read('/%sstarted.json' % build), gcs_async.read('/%sfinished.json' % build)]) futures.sort(key=lambda (job, build, s, f): (job, view_base.pad_numbers(build)), reverse=True) jobs = {} for job, build, started_fut, finished_fut in futures: started, finished = view_build.normalize_metadata(started_fut, finished_fut) jobs.setdefault(job, []).append((build, started, finished)) return jobs
def build_list(job_dir, before): """ Given a job dir, give a (partial) list of recent build started.json & finished.jsons. Args: job_dir: the GCS path holding the jobs Returns: a list of [(build, loc, started, finished)]. build is a string like "123", loc is the job directory and build, started/finished are either None or a dict of the finished.json, and a dict of {build: [issues...]} of xrefs. """ # pylint: disable=too-many-locals # /directory/ folders have a series of .txt files pointing at the correct location, # as a sort of fake symlink. indirect = '/directory/' in job_dir builds = get_build_numbers(job_dir, before, indirect) if indirect: # follow the indirect links build_symlinks = [ (build, gcs_async.read('%s%s.txt' % (job_dir, build))) for build in builds ] build_futures = [] for build, sym_fut in build_symlinks: redir = sym_fut.get_result() if redir and redir.startswith('gs://'): redir = redir[4:].strip() build_futures.append( (build, redir, gcs_async.read('%s/started.json' % redir), gcs_async.read('%s/finished.json' % redir))) else: build_futures = [ (build, '%s%s' % (job_dir, build), gcs_async.read('%s%s/started.json' % (job_dir, build)), gcs_async.read('%s%s/finished.json' % (job_dir, build))) for build in builds ] # This is done in parallel with waiting for GCS started/finished. build_refs = models.GHIssueDigest.find_xrefs_multi_async( [b[1] for b in build_futures]) output = [] for build, loc, started_future, finished_future in build_futures: started, finished = normalize_metadata(started_future, finished_future) output.append((str(build), loc, started, finished)) return output, build_refs.get_result()
def build_list(job_dir, before): ''' Given a job dir, give a (partial) list of recent build finished.jsons. Args: job_dir: the GCS path holding the jobs Returns: a list of [(build, finished)]. build is a string like "123", finished is either None or a dict of the finished.json. ''' latest_fut = gcs_async.read(job_dir + 'latest-build.txt') try: if 'pr-logs' in job_dir: raise ValueError('bad code path for PR build list') # If we have latest-build.txt, we can skip an expensive GCS ls call! latest_build = int(latest_fut.get_result()) if before: latest_build = int(before) - 1 else: # latest-build.txt has the most recent finished build. There might # be newer builds that have started but not finished. Probe for them. while gcs_async.read('%s%s/started.json' % (job_dir, latest_build + 1)).get_result(): latest_build += 1 builds = range(latest_build, max(0, latest_build - 40), -1) except (ValueError, TypeError): fstats = view_base.gcs_ls(job_dir) fstats.sort(key=lambda f: view_base.pad_numbers(f.filename), reverse=True) builds = [ os.path.basename(os.path.dirname(f.filename)) for f in fstats if f.is_dir ] if before and before in builds: builds = builds[builds.index(before) + 1:] builds = builds[:40] build_futures = [(build, gcs_async.read('%s%s/started.json' % (job_dir, build)), gcs_async.read('%s%s/finished.json' % (job_dir, build))) for build in builds] def resolve(future): res = future.get_result() if res: return json.loads(res) return [(str(build), resolve(started), resolve(finished)) for build, started, finished in build_futures]
def build_list(job_dir, before): ''' Given a job dir, give a (partial) list of recent build finished.jsons. Args: job_dir: the GCS path holding the jobs Returns: a list of [(build, finished)]. build is a string like "123", finished is either None or a dict of the finished.json. ''' # /directory/ folders have a series of .txt files pointing at the correct location, # as a sort of fake symlink. indirect = '/directory/' in job_dir builds = get_build_numbers(job_dir, before, indirect) if indirect: # follow the indirect links build_symlinks = [ (build, gcs_async.read('%s%s.txt' % (job_dir, build))) for build in builds ] build_futures = [] for build, sym_fut in build_symlinks: redir = sym_fut.get_result() if redir and redir.startswith('gs://'): redir = redir[4:].strip() build_futures.append( (build, redir, gcs_async.read('%s/started.json' % redir), gcs_async.read('%s/finished.json' % redir))) else: build_futures = [ (build, '%s%s' % (job_dir, build), gcs_async.read('%s%s/started.json' % (job_dir, build)), gcs_async.read('%s%s/finished.json' % (job_dir, build))) for build in builds ] def resolve(future): res = future.get_result() if res: return json.loads(res) return [(str(build), loc, resolve(started), resolve(finished)) for build, loc, started, finished in build_futures]
def build_details(build_dir): """ Collect information from a build directory. Args: build_dir: GCS path containing a build's results. Returns: started: value from started.json {'version': ..., 'timestamp': ...} finished: value from finished.json {'timestamp': ..., 'result': ...} failures: list of (name, duration, text) tuples build_log: a hilighted portion of errors in the build log. May be None. """ started_fut = gcs_async.read(build_dir + '/started.json') finished = gcs_async.read(build_dir + '/finished.json').get_result() started = started_fut.get_result() if finished and not started: started = 'null' if started and not finished: finished = 'null' elif not (started and finished): return started = json.loads(started) finished = json.loads(finished) failures = [] junit_paths = [ f.filename for f in view_base.gcs_ls('%s/artifacts' % build_dir) if re.match(r'junit_.*\.xml', os.path.basename(f.filename)) ] junit_futures = {} for f in junit_paths: junit_futures[gcs_async.read(f)] = f for future in junit_futures: junit = future.get_result() if junit is None: continue failures.extend(parse_junit(junit, junit_futures[future])) failures.sort() build_log = None if finished and finished.get('result') != 'SUCCESS' and len(failures) == 0: build_log = gcs_async.read(build_dir + '/build-log.txt').get_result() if build_log: build_log = log_parser.digest(build_log.decode('utf8', 'replace')) logging.info('fallback log parser emitted %d lines', build_log.count('\n')) return started, finished, failures, build_log
def get_woven_logs(log_files, pod, filters, objref_dict): lines = [] combined_lines = [] first_combined = "" pod_re = regex.wordRE(pod) # Produce a list of lines of all the selected logs for log_file in log_files: log = gcs_async.read(log_file).get_result() log = log.decode('utf8', 'replace') lines.extend(log.split('\n')) # Combine lines without timestamp into previous line, except if it comes at the # beginning of the file, in which case add it to the line with the first timestamp for line in lines: timestamp_re = regex.timestamp(line) if timestamp_re and timestamp_re.group(0): if not combined_lines: # add beginning of file to first timestamp line line = first_combined + line combined_lines.append(line) else: if not combined_lines: first_combined = first_combined + line else: combined_lines[-1] = combined_lines[-1] + line lines = sorted(combined_lines, key=regex.sub_timestamp) data = '\n'.join(lines) woven_logs = log_parser.digest(data, error_re=pod_re, filters=filters, objref_dict=objref_dict) return woven_logs
def build_details(build_dir): """ Collect information from a build directory. Args: build_dir: GCS path containing a build's results. Returns: started: value from started.json {'version': ..., 'timestamp': ...} finished: value from finished.json {'timestamp': ..., 'result': ...} failures: list of (name, duration, text) tuples build_log: a hilighted portion of errors in the build log. May be None. """ started_fut = gcs_async.read(build_dir + '/started.json') finished = gcs_async.read(build_dir + '/finished.json').get_result() started = started_fut.get_result() if finished and not started: started = 'null' if started and not finished: finished = 'null' elif not (started and finished): return started = json.loads(started) finished = json.loads(finished) failures = [] junit_paths = [f.filename for f in view_base.gcs_ls('%s/artifacts' % build_dir) if re.match(r'junit_.*\.xml', os.path.basename(f.filename))] junit_futures = {} for f in junit_paths: junit_futures[gcs_async.read(f)] = f for future in junit_futures: junit = future.get_result() if junit is None: continue failures.extend(parse_junit(junit, junit_futures[future])) failures.sort() build_log = None if finished and finished.get('result') != 'SUCCESS' and len(failures) == 0: build_log = gcs_async.read(build_dir + '/build-log.txt').get_result() if build_log: build_log = log_parser.digest(build_log.decode('utf8', 'replace')) logging.info('fallback log parser emitted %d lines', build_log.count('\n')) return started, finished, failures, build_log
def pr_builds(path, pr): """ Get information for all builds run by a PR. Args: pr: the PR number Returns: A dictionary of {job: [(build_number, started_json, finished.json)]} """ jobs_dirs_fut = gcs_async.listdirs('%s/%s%s' % (PR_PREFIX, path, pr)) print '%s/%s%s' % (PR_PREFIX, path, pr) def base(path): return os.path.basename(os.path.dirname(path)) jobs_futures = [(job, gcs_async.listdirs(job)) for job in jobs_dirs_fut.get_result()] futures = [] for job, builds_fut in jobs_futures: for build in builds_fut.get_result(): futures.append([ base(job), base(build), gcs_async.read('/%sstarted.json' % build), gcs_async.read('/%sfinished.json' % build) ]) futures.sort(key=lambda (job, build, s, f): (job, view_base.pad_numbers(build)), reverse=True) jobs = {} for job, build, started_fut, finished_fut in futures: started = started_fut.get_result() finished = finished_fut.get_result() if started is not None: started = json.loads(started) if finished is not None: finished = json.loads(finished) jobs.setdefault(job, []).append((build, started, finished)) return jobs
def build_details(build_dir): """ Collect information from a build directory. Args: build_dir: GCS path containing a build's results. Returns: started: value from started.json {'version': ..., 'timestamp': ...} finished: value from finished.json {'timestamp': ..., 'result': ...} failures: list of (name, duration, text) tuples build_log: a hilighted portion of errors in the build log. May be None. """ started_fut = gcs_async.read(build_dir + '/started.json') finished = gcs_async.read(build_dir + '/finished.json').get_result() started = started_fut.get_result() if finished and not started: started = 'null' if started and not finished: finished = 'null' elif not (started and finished): return started = json.loads(started) finished = json.loads(finished) failures = [] junit_paths = [ f.filename for f in view_base.gcs_ls('%s/artifacts' % build_dir) if re.match(r'junit_.*\.xml', os.path.basename(f.filename)) ] junit_futures = {} for f in junit_paths: junit_futures[gcs_async.read(f)] = f for future in junit_futures: junit = future.get_result() if not junit: continue failures.extend(parse_junit(junit, junit_futures[future])) failures.sort() return started, finished, failures
def parse_log_file(log_filename, pod, filters=None, make_dict=False, objref_dict=None): """Based on make_dict, either returns the objref_dict or the parsed log file""" log = gcs_async.read(log_filename).get_result() if log is None: return None pod_re = regex.wordRE(pod) if make_dict: return kubelet_parser.make_dict(log.decode('utf8', 'replace'), pod_re) else: return log_parser.digest(log.decode('utf8', 'replace'), error_re=pod_re, filters=filters, objref_dict=objref_dict)
def get(self): # let's lock this down to build logs for now. path = self.request.get('path') if not re.match(r'^[-\w/.]+$', path): self.abort(403) if not path.endswith('/build-log.txt'): self.abort(403) content = gcs_async.read(path).get_result() # lazy XSS prevention. # doesn't work on terrible browsers that do content sniffing (ancient IE). self.response.headers['Content-Type'] = 'text/plain' self.response.write(content)
def build_details(build_dir): """ Collect information from a build directory. Args: build_dir: GCS path containing a build's results. Returns: started: value from started.json {'version': ..., 'timestamp': ...} finished: value from finished.json {'timestamp': ..., 'result': ...} failures: list of (name, duration, text) tuples build_log: a hilighted portion of errors in the build log. May be None. """ started_fut = gcs_async.read(build_dir + '/started.json') finished = gcs_async.read(build_dir + '/finished.json').get_result() started = started_fut.get_result() if finished and not started: started = 'null' if started and not finished: finished = 'null' elif not (started and finished): return started = json.loads(started) finished = json.loads(finished) failures = [] junit_paths = [f.filename for f in view_base.gcs_ls('%s/artifacts' % build_dir) if re.match(r'junit_.*\.xml', os.path.basename(f.filename))] junit_futures = {} for f in junit_paths: junit_futures[gcs_async.read(f)] = f for future in junit_futures: junit = future.get_result() if not junit: continue failures.extend(parse_junit(junit, junit_futures[future])) failures.sort() return started, finished, failures
def pr_builds(path, pr): """ Get information for all builds run by a PR. Args: pr: the PR number Returns: A dictionary of {job: [(build_number, started_json, finished.json)]} """ jobs_dirs_fut = gcs_async.listdirs('%s/%s%s' % (PR_PREFIX, path, pr)) print '%s/%s%s' % (PR_PREFIX, path, pr) def base(path): return os.path.basename(os.path.dirname(path)) jobs_futures = [(job, gcs_async.listdirs(job)) for job in jobs_dirs_fut.get_result()] futures = [] for job, builds_fut in jobs_futures: for build in builds_fut.get_result(): futures.append([ base(job), base(build), gcs_async.read('/%sstarted.json' % build), gcs_async.read('/%sfinished.json' % build)]) futures.sort(key=lambda (job, build, s, f): (job, view_base.pad_numbers(build)), reverse=True) jobs = {} for job, build, started_fut, finished_fut in futures: started = started_fut.get_result() finished = finished_fut.get_result() if started is not None: started = json.loads(started) if finished is not None: finished = json.loads(finished) jobs.setdefault(job, []).append((build, started, finished)) return jobs
def parse_log_file(log_filename, pod, filters=None, make_dict=False, objref_dict=None): """Based on make_dict, either returns the objref_dict or the parsed log file""" log = gcs_async.read(log_filename).get_result() if log is None: return {}, False if make_dict else None if pod: bold_re = regex.wordRE(pod) else: bold_re = regex.error_re if objref_dict is None: objref_dict = {} if make_dict and pod: return kubelet_parser.make_dict(log.decode('utf8', 'replace'), bold_re, objref_dict) else: return log_parser.digest(log.decode('utf8', 'replace'), error_re=bold_re, filters=filters, objref_dict=objref_dict)
def parse_kubelet(pod, junit, build_dir, filters): junit_file = junit + ".xml" tmps = [f.filename for f in gcs_ls('%s/artifacts' % build_dir) if re.match(r'.*/tmp-node.*', f.filename)] junit_regex = r".*" + junit_file + r".*" kubelet_filename = "" for folder in tmps: tmp_contents = [f.filename for f in gcs_ls(folder)] for f in tmp_contents: if re.match(junit_regex, f): for file in tmp_contents: if re.match(r'.*kubelet\.log', file): kubelet_filename = file if kubelet_filename == "": return False kubelet_log = gcs_async.read(kubelet_filename).get_result() if kubelet_log: pod_re = regex.wordRE(pod) kubelet_log = log_parser.digest(kubelet_log.decode('utf8', 'replace'), error_re=pod_re, filters=filters) return kubelet_log
def test_read(self): write('/foo/bar', 'test data') self.assertEqual(gcs_async.read('/foo/bar').get_result(), 'test data') self.assertEqual(gcs_async.read('/foo/quux').get_result(), None)
def get_build_log(build_dir): build_log = gcs_async.read(build_dir + '/build-log.txt').get_result() if build_log: return log_parser.digest(build_log)
def get(self, prefix, job, build): # pylint: disable=too-many-locals if prefix.endswith('/directory'): # redirect directory requests link = gcs_async.read('/%s/%s/%s.txt' % (prefix, job, build)).get_result() if link and link.startswith('gs://'): self.redirect('/build/' + link.replace('gs://', '')) return job_dir = '/%s/%s/' % (prefix, job) testgrid_query = testgrid.path_to_query(job_dir) build_dir = job_dir + build started, finished, metadata, results = build_details(build_dir) if started is None and finished is None: logging.warning('unable to load %s', build_dir) self.render( 'build_404.html', dict(build_dir=build_dir, job_dir=job_dir, job=job, build=build)) self.response.set_status(404) return want_build_log = False build_log = '' build_log_src = None if 'log' in self.request.params or (not finished) or \ (finished and finished.get('result') != 'SUCCESS' and len(results['failed']) <= 1): want_build_log = True build_log = get_build_log(build_dir) pr, pr_path, pr_digest = None, None, None repo = '%s/%s' % (self.app.config['default_org'], self.app.config['default_repo']) spyglass_link = '' external_config = get_build_config(prefix, self.app.config) if external_config is not None: if external_config.get('spyglass'): spyglass_link = 'https://' + external_config['prow_url'] + '/view/gcs/' + build_dir if '/pull/' in prefix: pr, pr_path, pr_digest, repo = get_pr_info(prefix, self.app.config) if want_build_log and not build_log: build_log, build_log_src = get_running_build_log(job, build, external_config["prow_url"]) # 'version' might be in either started or finished. # prefer finished. if finished and 'version' in finished: version = finished['version'] else: version = started and started.get('version') commit = version and version.split('+')[-1] issues = list(models.GHIssueDigest.find_xrefs(build_dir)) # openshift does not have a pull string because the entrypoint does not # set it into started ref_string = "" if 'repos' in metadata and metadata['repos']: if repo in metadata['repos']: ref_string = metadata['repos'][repo] del metadata['repos'][repo] if len(ref_string) == 0 and started and 'pull' in started: ref_string = started['pull'] refs = [] if len(ref_string) > 0: for ref in ref_string.split(','): x = ref.split(':', 1) if len(x) == 2: refs.append((x[0], x[1])) else: refs.append((x[0], '')) work_namespace = "" if 'work-namespace' in metadata: work_namespace = metadata['work-namespace'] del metadata['work-namespace'] self.render('build.html', dict( job_dir=job_dir, build_dir=build_dir, job=job, build=build, commit=commit, started=started, finished=finished, metadata=metadata, res=results, refs=refs, work_namespace=work_namespace, build_log=build_log, build_log_src=build_log_src, issues=issues, repo=repo, pr_path=pr_path, pr=pr, pr_digest=pr_digest, testgrid_query=testgrid_query))
def get(self, prefix, job, build): # pylint: disable=too-many-locals if prefix.endswith('/directory'): # redirect directory requests link = gcs_async.read('/%s/%s/%s.txt' % (prefix, job, build)).get_result() if link and link.startswith('gs://'): self.redirect('/build/' + link.replace('gs://', '')) return job_dir = '/%s/%s/' % (prefix, job) testgrid_query = testgrid.path_to_query(job_dir) build_dir = job_dir + build started, finished, results = build_details(build_dir) if started is None and finished is None: logging.warning('unable to load %s', build_dir) self.render( 'build_404.html', dict(build_dir=build_dir, job_dir=job_dir, job=job, build=build)) self.response.set_status(404) return want_build_log = False build_log = '' build_log_src = None if 'log' in self.request.params or (not finished) or \ (finished and finished.get('result') != 'SUCCESS' and len(results['failed']) <= 1): want_build_log = True build_log = get_build_log(build_dir) pr, pr_path, pr_digest = None, None, None repo = '%s/%s' % (self.app.config['default_org'], self.app.config['default_repo']) external_config = get_build_config(prefix, self.app.config) if external_config is not None: if '/pull/' in prefix: pr, pr_path, pr_digest, repo = get_pr_info(prefix, self.app.config) if want_build_log and not build_log: build_log, build_log_src = get_running_build_log(job, build, external_config["prow_url"]) # 'version' might be in either started or finished. # prefer finished. version = finished and finished.get('version') or started and started.get('version') commit = version and version.split('+')[-1] issues = list(models.GHIssueDigest.find_xrefs(build_dir)) refs = [] if started and started.get('pull'): for ref in started['pull'].split(','): x = ref.split(':', 1) if len(x) == 2: refs.append((x[0], x[1])) else: refs.append((x[0], '')) self.render('build.html', dict( job_dir=job_dir, build_dir=build_dir, job=job, build=build, commit=commit, started=started, finished=finished, res=results, refs=refs, build_log=build_log, build_log_src=build_log_src, issues=issues, repo=repo, pr_path=pr_path, pr=pr, pr_digest=pr_digest, testgrid_query=testgrid_query))
def get(self, prefix, job, build): # pylint: disable=too-many-locals if prefix.endswith('/directory'): # redirect directory requests link = gcs_async.read('/%s/%s/%s.txt' % (prefix, job, build)).get_result() if link and link.startswith('gs://'): self.redirect('/build/' + link.replace('gs://', '')) return job_dir = '/%s/%s/' % (prefix, job) testgrid_query = testgrid.path_to_query(job_dir) build_dir = job_dir + build started, finished, results = build_details(build_dir) if started is None and finished is None: logging.warning('unable to load %s', build_dir) self.render( 'build_404.html', dict(build_dir=build_dir, job_dir=job_dir, job=job, build=build)) self.response.set_status(404) return want_build_log = False build_log = '' build_log_src = None if 'log' in self.request.params or (not finished) or \ (finished and finished.get('result') != 'SUCCESS' and len(results['failed']) <= 1): want_build_log = True build_log = get_build_log(build_dir) pr, pr_path, pr_digest = None, None, None repo = '%s/%s' % (self.app.config['default_org'], self.app.config['default_repo']) external_config = get_build_config(prefix, self.app.config) if external_config is not None: if '/pull/' in prefix: pr, pr_path, pr_digest, repo = get_pr_info( prefix, self.app.config) if want_build_log and not build_log: build_log, build_log_src = get_running_build_log( job, build, external_config["prow_url"]) # 'version' might be in either started or finished. # prefer finished. version = finished and finished.get( 'version') or started and started.get('version') commit = version and version.split('+')[-1] issues = list(models.GHIssueDigest.find_xrefs(build_dir)) refs = [] if started and started.get('pull'): for ref in started['pull'].split(','): x = ref.split(':', 1) if len(x) == 2: refs.append((x[0], x[1])) else: refs.append((x[0], '')) self.render( 'build.html', dict(job_dir=job_dir, build_dir=build_dir, job=job, build=build, commit=commit, started=started, finished=finished, res=results, refs=refs, build_log=build_log, build_log_src=build_log_src, issues=issues, repo=repo, pr_path=pr_path, pr=pr, pr_digest=pr_digest, testgrid_query=testgrid_query))