def build_list(job_dir, before):
'''
Given a job dir, give a (partial) list of recent build
finished.jsons.
Args:
job_dir: the GCS path holding the jobs
Returns:
a list of [(build, loc, started, finished)].
build is a string like "123",
loc is the job directory and build,
started/finished are either None or a dict of the finished.json,
and a dict of {build: [issues...]} of xrefs.
'''
# /directory/ folders have a series of .txt files pointing at the correct location,
# as a sort of fake symlink.
indirect = '/directory/' in job_dir
builds = get_build_numbers(job_dir, before, indirect)
if indirect:
# follow the indirect links
build_symlinks = [
(build,
gcs_async.read('%s%s.txt' % (job_dir, build)))
for build in builds
]
build_futures = []
for build, sym_fut in build_symlinks:
redir = sym_fut.get_result()
if redir and redir.startswith('gs://'):
redir = redir[4:].strip()
build_futures.append(
(build, redir,
gcs_async.read('%s/started.json' % redir),
gcs_async.read('%s/finished.json' % redir)))
else:
build_futures = [
(build, '%s%s' % (job_dir, build),
gcs_async.read('%s%s/started.json' % (job_dir, build)),
gcs_async.read('%s%s/finished.json' % (job_dir, build)))
for build in builds
]
refs = {}
output = []
for build, loc, started_future, finished_future in build_futures:
started, finished, metadata = normalize_metadata(started_future, finished_future)
issues = []
if "repos" in metadata and metadata["repos"]:
for repo in metadata['repos']:
for ref in metadata['repos'][repo].split(','):
x = ref.split(':', 1)
if len(x) == 2 and x[0].isdigit():
issues.append({"number": int(x[0]), "title": "", "url": "https://github.com/%s/pull/%s" % (repo, x[0]) })
refs[loc] = issues
output.append((str(build), loc, started, finished))
return output, refs
def pr_builds(path):
"""Return {job: [(build, {started.json}, {finished.json})]} for each job under gcs path."""
jobs_dirs_fut = gcs_async.listdirs(path)
def base(path):
return os.path.basename(os.path.dirname(path))
jobs_futures = [(job, gcs_async.listdirs(job)) for job in jobs_dirs_fut.get_result()]
futures = []
for job, builds_fut in jobs_futures:
for build in builds_fut.get_result():
futures.append([
base(job),
base(build),
gcs_async.read('/%sstarted.json' % build),
gcs_async.read('/%sfinished.json' % build)])
futures.sort(key=lambda (job, build, s, f): (job, view_base.pad_numbers(build)), reverse=True)
jobs = {}
for job, build, started_fut, finished_fut in futures:
started = started_fut.get_result()
finished = finished_fut.get_result()
if started is not None:
started = json.loads(started)
if finished is not None:
finished = json.loads(finished)
jobs.setdefault(job, []).append((build, started, finished))
return jobs
def build_details(build_dir):
"""
Collect information from a build directory.
Args:
build_dir: GCS path containing a build's results.
Returns:
started: value from started.json {'version': ..., 'timestamp': ...}
finished: value from finished.json {'timestamp': ..., 'result': ...}
results: {total: int,
failed: [(name, duration, text)...],
skipped: [name...],
passed: [name...]}
"""
started_fut = gcs_async.read(build_dir + '/started.json')
finished = gcs_async.read(build_dir + '/finished.json').get_result()
started = started_fut.get_result()
if finished and not started:
started = 'null'
if started and not finished:
finished = 'null'
elif not (started and finished):
return
started = json.loads(started)
finished = json.loads(finished)
junit_paths = [f.filename for f in view_base.gcs_ls('%s/artifacts' % build_dir)
if re.match(r'junit_.*\.xml', os.path.basename(f.filename))]
junit_futures = {f: gcs_async.read(f) for f in junit_paths}
parser = JUnitParser()
for path, future in junit_futures.iteritems():
parser.parse_xml(future.get_result(), path)
return started, finished, parser.get_results()
def get_build_numbers(job_dir, before, indirect):
try:
if 'pr-logs' in job_dir and not indirect:
raise ValueError('bad code path for PR build list')
# If we have latest-build.txt, we can skip an expensive GCS ls call!
if before:
latest_build = int(before) - 1
else:
latest_build = int(gcs_async.read(job_dir + 'latest-build.txt').get_result())
# latest-build.txt has the most recent finished build. There might
# be newer builds that have started but not finished. Probe for them.
suffix = '/started.json' if not indirect else '.txt'
while gcs_async.read('%s%s%s' % (job_dir, latest_build + 1, suffix)).get_result():
latest_build += 1
return range(latest_build, max(0, latest_build - 40), -1)
except (ValueError, TypeError):
fstats = view_base.gcs_ls(job_dir)
fstats.sort(key=lambda f: view_base.pad_numbers(f.filename),
reverse=True)
if indirect:
# find numbered builds
builds = [re.search(r'/(\d*)\.txt$', f.filename)
for f in fstats if not f.is_dir]
builds = [m.group(1) for m in builds if m]
else:
builds = [os.path.basename(os.path.dirname(f.filename))
for f in fstats if f.is_dir]
if before and before in builds:
builds = builds[builds.index(before) + 1:]
return builds[:40]
def build_details(build_dir):
"""
Collect information from a build directory.
Args:
build_dir: GCS path containing a build's results.
Returns:
started: value from started.json {'version': ..., 'timestamp': ...}
finished: value from finished.json {'timestamp': ..., 'result': ...}
results: {total: int,
failed: [(name, duration, text)...],
skipped: [name...],
passed: [name...]}
"""
started, finished, metadata = normalize_metadata(
gcs_async.read(build_dir + '/started.json'),
gcs_async.read(build_dir + '/finished.json')
)
if started is None and finished is None:
return started, finished, {}, None
junit_paths = [f.filename for f in view_base.gcs_ls_recursive('%s/artifacts' % build_dir)
if f.filename.endswith('.xml')]
junit_futures = {f: gcs_async.read(f) for f in junit_paths}
parser = JUnitParser()
for path, future in junit_futures.iteritems():
parser.parse_xml(future.get_result(), path)
return started, finished, metadata, parser.get_results()
def build_details(build_dir):
"""
Collect information from a build directory.
Args:
build_dir: GCS path containing a build's results.
Returns:
started: value from started.json {'version': ..., 'timestamp': ...}
finished: value from finished.json {'timestamp': ..., 'result': ...}
results: {total: int,
failed: [(name, duration, text)...],
skipped: [name...],
passed: [name...]}
"""
started, finished = normalize_metadata(
gcs_async.read(build_dir + '/started.json'),
gcs_async.read(build_dir + '/finished.json')
)
if started is None and finished is None:
return started, finished, None
junit_paths = [f.filename for f in view_base.gcs_ls_recursive('%s/artifacts' % build_dir)
if f.filename.endswith('.xml')]
junit_futures = {f: gcs_async.read(f) for f in junit_paths}
parser = JUnitParser()
for path, future in junit_futures.iteritems():
parser.parse_xml(future.get_result(), path)
return started, finished, parser.get_results()
def get_build_numbers(job_dir, before, indirect):
try:
if '/pull/' in job_dir and not indirect:
raise ValueError('bad code path for PR build list')
# If we have latest-build.txt, we can skip an expensive GCS ls call!
if before:
latest_build = int(before) - 1
else:
latest_build = int(gcs_async.read(job_dir + 'latest-build.txt').get_result())
# latest-build.txt has the most recent finished build. There might
# be newer builds that have started but not finished. Probe for them.
suffix = '/started.json' if not indirect else '.txt'
while gcs_async.read('%s%s%s' % (job_dir, latest_build + 1, suffix)).get_result():
latest_build += 1
return range(latest_build, max(0, latest_build - 40), -1)
except (ValueError, TypeError):
fstats = view_base.gcs_ls(job_dir)
fstats.sort(key=lambda f: view_base.pad_numbers(f.filename),
reverse=True)
if indirect:
# find numbered builds
builds = [re.search(r'/(\d*)\.txt$', f.filename)
for f in fstats if not f.is_dir]
builds = [m.group(1) for m in builds if m]
else:
builds = [os.path.basename(os.path.dirname(f.filename))
for f in fstats if f.is_dir]
if before and before in builds:
builds = builds[builds.index(before) + 1:]
return builds[:40]
def pr_builds(path):
"""Return {job: [(build, {started.json}, {finished.json})]} for each job under gcs path."""
jobs_dirs_fut = gcs_async.listdirs(path)
def base(path):
return os.path.basename(os.path.dirname(path))
jobs_futures = [(job, gcs_async.listdirs(job)) for job in jobs_dirs_fut.get_result()]
futures = []
for job, builds_fut in jobs_futures:
for build in builds_fut.get_result():
futures.append([
base(job),
base(build),
gcs_async.read('/%sstarted.json' % build),
gcs_async.read('/%sfinished.json' % build)])
futures.sort(key=lambda (job, build, s, f): (job, view_base.pad_numbers(build)), reverse=True)
jobs = {}
for job, build, started_fut, finished_fut in futures:
started, finished = view_build.normalize_metadata(started_fut, finished_fut)
jobs.setdefault(job, []).append((build, started, finished))
return jobs
def build_list(job_dir, before):
"""
Given a job dir, give a (partial) list of recent build
started.json & finished.jsons.
Args:
job_dir: the GCS path holding the jobs
Returns:
a list of [(build, loc, started, finished)].
build is a string like "123",
loc is the job directory and build,
started/finished are either None or a dict of the finished.json,
and a dict of {build: [issues...]} of xrefs.
"""
# pylint: disable=too-many-locals
# /directory/ folders have a series of .txt files pointing at the correct location,
# as a sort of fake symlink.
indirect = '/directory/' in job_dir
builds = get_build_numbers(job_dir, before, indirect)
if indirect:
# follow the indirect links
build_symlinks = [
(build,
gcs_async.read('%s%s.txt' % (job_dir, build)))
for build in builds
]
build_futures = []
for build, sym_fut in build_symlinks:
redir = sym_fut.get_result()
if redir and redir.startswith('gs://'):
redir = redir[4:].strip()
build_futures.append(
(build, redir,
gcs_async.read('%s/started.json' % redir),
gcs_async.read('%s/finished.json' % redir)))
else:
build_futures = [
(build, '%s%s' % (job_dir, build),
gcs_async.read('%s%s/started.json' % (job_dir, build)),
gcs_async.read('%s%s/finished.json' % (job_dir, build)))
for build in builds
]
# This is done in parallel with waiting for GCS started/finished.
build_refs = models.GHIssueDigest.find_xrefs_multi_async(
[b[1] for b in build_futures])
output = []
for build, loc, started_future, finished_future in build_futures:
started, finished = normalize_metadata(started_future, finished_future)
output.append((str(build), loc, started, finished))
return output, build_refs.get_result()
def build_list(job_dir, before):
"""
Given a job dir, give a (partial) list of recent build
started.json & finished.jsons.
Args:
job_dir: the GCS path holding the jobs
Returns:
a list of [(build, loc, started, finished)].
build is a string like "123",
loc is the job directory and build,
started/finished are either None or a dict of the finished.json,
and a dict of {build: [issues...]} of xrefs.
"""
# pylint: disable=too-many-locals
# /directory/ folders have a series of .txt files pointing at the correct location,
# as a sort of fake symlink.
indirect = '/directory/' in job_dir
builds = get_build_numbers(job_dir, before, indirect)
if indirect:
# follow the indirect links
build_symlinks = [
(build,
gcs_async.read('%s%s.txt' % (job_dir, build)))
for build in builds
]
build_futures = []
for build, sym_fut in build_symlinks:
redir = sym_fut.get_result()
if redir and redir.startswith('gs://'):
redir = redir[4:].strip()
build_futures.append(
(build, redir,
gcs_async.read('%s/started.json' % redir),
gcs_async.read('%s/finished.json' % redir)))
else:
build_futures = [
(build, '%s%s' % (job_dir, build),
gcs_async.read('%s%s/started.json' % (job_dir, build)),
gcs_async.read('%s%s/finished.json' % (job_dir, build)))
for build in builds
]
# This is done in parallel with waiting for GCS started/finished.
build_refs = models.GHIssueDigest.find_xrefs_multi_async(
[b[1] for b in build_futures])
output = []
for build, loc, started_future, finished_future in build_futures:
started, finished = normalize_metadata(started_future, finished_future)
output.append((str(build), loc, started, finished))
return output, build_refs.get_result()
def build_list(job_dir, before):
'''
Given a job dir, give a (partial) list of recent build
finished.jsons.
Args:
job_dir: the GCS path holding the jobs
Returns:
a list of [(build, finished)]. build is a string like "123",
finished is either None or a dict of the finished.json.
'''
latest_fut = gcs_async.read(job_dir + 'latest-build.txt')
try:
if 'pr-logs' in job_dir:
raise ValueError('bad code path for PR build list')
# If we have latest-build.txt, we can skip an expensive GCS ls call!
latest_build = int(latest_fut.get_result())
if before:
latest_build = int(before) - 1
else:
# latest-build.txt has the most recent finished build. There might
# be newer builds that have started but not finished. Probe for them.
while gcs_async.read('%s%s/started.json' %
(job_dir, latest_build + 1)).get_result():
latest_build += 1
builds = range(latest_build, max(0, latest_build - 40), -1)
except (ValueError, TypeError):
fstats = view_base.gcs_ls(job_dir)
fstats.sort(key=lambda f: view_base.pad_numbers(f.filename),
reverse=True)
builds = [
os.path.basename(os.path.dirname(f.filename)) for f in fstats
if f.is_dir
]
if before and before in builds:
builds = builds[builds.index(before) + 1:]
builds = builds[:40]
build_futures = [(build,
gcs_async.read('%s%s/started.json' % (job_dir, build)),
gcs_async.read('%s%s/finished.json' % (job_dir, build)))
for build in builds]
def resolve(future):
res = future.get_result()
if res:
return json.loads(res)
return [(str(build), resolve(started), resolve(finished))
for build, started, finished in build_futures]
def build_list(job_dir, before):
'''
Given a job dir, give a (partial) list of recent build
finished.jsons.
Args:
job_dir: the GCS path holding the jobs
Returns:
a list of [(build, finished)]. build is a string like "123",
finished is either None or a dict of the finished.json.
'''
# /directory/ folders have a series of .txt files pointing at the correct location,
# as a sort of fake symlink.
indirect = '/directory/' in job_dir
builds = get_build_numbers(job_dir, before, indirect)
if indirect:
# follow the indirect links
build_symlinks = [
(build,
gcs_async.read('%s%s.txt' % (job_dir, build)))
for build in builds
]
build_futures = []
for build, sym_fut in build_symlinks:
redir = sym_fut.get_result()
if redir and redir.startswith('gs://'):
redir = redir[4:].strip()
build_futures.append(
(build, redir,
gcs_async.read('%s/started.json' % redir),
gcs_async.read('%s/finished.json' % redir)))
else:
build_futures = [
(build, '%s%s' % (job_dir, build),
gcs_async.read('%s%s/started.json' % (job_dir, build)),
gcs_async.read('%s%s/finished.json' % (job_dir, build)))
for build in builds
]
def resolve(future):
res = future.get_result()
if res:
return json.loads(res)
return [(str(build), loc, resolve(started), resolve(finished))
for build, loc, started, finished in build_futures]
def build_details(build_dir):
"""
Collect information from a build directory.
Args:
build_dir: GCS path containing a build's results.
Returns:
started: value from started.json {'version': ..., 'timestamp': ...}
finished: value from finished.json {'timestamp': ..., 'result': ...}
failures: list of (name, duration, text) tuples
build_log: a hilighted portion of errors in the build log. May be None.
"""
started_fut = gcs_async.read(build_dir + '/started.json')
finished = gcs_async.read(build_dir + '/finished.json').get_result()
started = started_fut.get_result()
if finished and not started:
started = 'null'
if started and not finished:
finished = 'null'
elif not (started and finished):
return
started = json.loads(started)
finished = json.loads(finished)
failures = []
junit_paths = [
f.filename for f in view_base.gcs_ls('%s/artifacts' % build_dir)
if re.match(r'junit_.*\.xml', os.path.basename(f.filename))
]
junit_futures = {}
for f in junit_paths:
junit_futures[gcs_async.read(f)] = f
for future in junit_futures:
junit = future.get_result()
if junit is None:
continue
failures.extend(parse_junit(junit, junit_futures[future]))
failures.sort()
build_log = None
if finished and finished.get('result') != 'SUCCESS' and len(failures) == 0:
build_log = gcs_async.read(build_dir + '/build-log.txt').get_result()
if build_log:
build_log = log_parser.digest(build_log.decode('utf8', 'replace'))
logging.info('fallback log parser emitted %d lines',
build_log.count('\n'))
return started, finished, failures, build_log
def build_list(job_dir, before):
'''
Given a job dir, give a (partial) list of recent build
finished.jsons.
Args:
job_dir: the GCS path holding the jobs
Returns:
a list of [(build, finished)]. build is a string like "123",
finished is either None or a dict of the finished.json.
'''
# /directory/ folders have a series of .txt files pointing at the correct location,
# as a sort of fake symlink.
indirect = '/directory/' in job_dir
builds = get_build_numbers(job_dir, before, indirect)
if indirect:
# follow the indirect links
build_symlinks = [
(build,
gcs_async.read('%s%s.txt' % (job_dir, build)))
for build in builds
]
build_futures = []
for build, sym_fut in build_symlinks:
redir = sym_fut.get_result()
if redir and redir.startswith('gs://'):
redir = redir[4:].strip()
build_futures.append(
(build, redir,
gcs_async.read('%s/started.json' % redir),
gcs_async.read('%s/finished.json' % redir)))
else:
build_futures = [
(build, '%s%s' % (job_dir, build),
gcs_async.read('%s%s/started.json' % (job_dir, build)),
gcs_async.read('%s%s/finished.json' % (job_dir, build)))
for build in builds
]
def resolve(future):
res = future.get_result()
if res:
return json.loads(res)
return [(str(build), loc, resolve(started), resolve(finished))
for build, loc, started, finished in build_futures]
def get_woven_logs(log_files, pod, filters, objref_dict):
lines = []
combined_lines = []
first_combined = ""
pod_re = regex.wordRE(pod)
# Produce a list of lines of all the selected logs
for log_file in log_files:
log = gcs_async.read(log_file).get_result()
log = log.decode('utf8', 'replace')
lines.extend(log.split('\n'))
# Combine lines without timestamp into previous line, except if it comes at the
# beginning of the file, in which case add it to the line with the first timestamp
for line in lines:
timestamp_re = regex.timestamp(line)
if timestamp_re and timestamp_re.group(0):
if not combined_lines:
# add beginning of file to first timestamp line
line = first_combined + line
combined_lines.append(line)
else:
if not combined_lines:
first_combined = first_combined + line
else:
combined_lines[-1] = combined_lines[-1] + line
lines = sorted(combined_lines, key=regex.sub_timestamp)
data = '\n'.join(lines)
woven_logs = log_parser.digest(data,
error_re=pod_re,
filters=filters,
objref_dict=objref_dict)
return woven_logs
def get_woven_logs(log_files, pod, filters, objref_dict):
lines = []
combined_lines = []
first_combined = ""
pod_re = regex.wordRE(pod)
# Produce a list of lines of all the selected logs
for log_file in log_files:
log = gcs_async.read(log_file).get_result()
log = log.decode('utf8', 'replace')
lines.extend(log.split('\n'))
# Combine lines without timestamp into previous line, except if it comes at the
# beginning of the file, in which case add it to the line with the first timestamp
for line in lines:
timestamp_re = regex.timestamp(line)
if timestamp_re and timestamp_re.group(0):
if not combined_lines:
# add beginning of file to first timestamp line
line = first_combined + line
combined_lines.append(line)
else:
if not combined_lines:
first_combined = first_combined + line
else:
combined_lines[-1] = combined_lines[-1] + line
lines = sorted(combined_lines, key=regex.sub_timestamp)
data = '\n'.join(lines)
woven_logs = log_parser.digest(data, error_re=pod_re,
filters=filters, objref_dict=objref_dict)
return woven_logs
def build_details(build_dir):
"""
Collect information from a build directory.
Args:
build_dir: GCS path containing a build's results.
Returns:
started: value from started.json {'version': ..., 'timestamp': ...}
finished: value from finished.json {'timestamp': ..., 'result': ...}
failures: list of (name, duration, text) tuples
build_log: a hilighted portion of errors in the build log. May be None.
"""
started_fut = gcs_async.read(build_dir + '/started.json')
finished = gcs_async.read(build_dir + '/finished.json').get_result()
started = started_fut.get_result()
if finished and not started:
started = 'null'
if started and not finished:
finished = 'null'
elif not (started and finished):
return
started = json.loads(started)
finished = json.loads(finished)
failures = []
junit_paths = [f.filename for f in view_base.gcs_ls('%s/artifacts' % build_dir)
if re.match(r'junit_.*\.xml', os.path.basename(f.filename))]
junit_futures = {}
for f in junit_paths:
junit_futures[gcs_async.read(f)] = f
for future in junit_futures:
junit = future.get_result()
if junit is None:
continue
failures.extend(parse_junit(junit, junit_futures[future]))
failures.sort()
build_log = None
if finished and finished.get('result') != 'SUCCESS' and len(failures) == 0:
build_log = gcs_async.read(build_dir + '/build-log.txt').get_result()
if build_log:
build_log = log_parser.digest(build_log.decode('utf8', 'replace'))
logging.info('fallback log parser emitted %d lines',
build_log.count('\n'))
return started, finished, failures, build_log
def pr_builds(path, pr):
"""
Get information for all builds run by a PR.
Args:
pr: the PR number
Returns:
A dictionary of {job: [(build_number, started_json, finished.json)]}
"""
jobs_dirs_fut = gcs_async.listdirs('%s/%s%s' % (PR_PREFIX, path, pr))
print '%s/%s%s' % (PR_PREFIX, path, pr)
def base(path):
return os.path.basename(os.path.dirname(path))
jobs_futures = [(job, gcs_async.listdirs(job))
for job in jobs_dirs_fut.get_result()]
futures = []
for job, builds_fut in jobs_futures:
for build in builds_fut.get_result():
futures.append([
base(job),
base(build),
gcs_async.read('/%sstarted.json' % build),
gcs_async.read('/%sfinished.json' % build)
])
futures.sort(key=lambda (job, build, s, f):
(job, view_base.pad_numbers(build)),
reverse=True)
jobs = {}
for job, build, started_fut, finished_fut in futures:
started = started_fut.get_result()
finished = finished_fut.get_result()
if started is not None:
started = json.loads(started)
if finished is not None:
finished = json.loads(finished)
jobs.setdefault(job, []).append((build, started, finished))
return jobs
def build_details(build_dir):
"""
Collect information from a build directory.
Args:
build_dir: GCS path containing a build's results.
Returns:
started: value from started.json {'version': ..., 'timestamp': ...}
finished: value from finished.json {'timestamp': ..., 'result': ...}
failures: list of (name, duration, text) tuples
build_log: a hilighted portion of errors in the build log. May be None.
"""
started_fut = gcs_async.read(build_dir + '/started.json')
finished = gcs_async.read(build_dir + '/finished.json').get_result()
started = started_fut.get_result()
if finished and not started:
started = 'null'
if started and not finished:
finished = 'null'
elif not (started and finished):
return
started = json.loads(started)
finished = json.loads(finished)
failures = []
junit_paths = [
f.filename for f in view_base.gcs_ls('%s/artifacts' % build_dir)
if re.match(r'junit_.*\.xml', os.path.basename(f.filename))
]
junit_futures = {}
for f in junit_paths:
junit_futures[gcs_async.read(f)] = f
for future in junit_futures:
junit = future.get_result()
if not junit:
continue
failures.extend(parse_junit(junit, junit_futures[future]))
failures.sort()
return started, finished, failures
def parse_log_file(log_filename, pod, filters=None, make_dict=False, objref_dict=None):
"""Based on make_dict, either returns the objref_dict or the parsed log file"""
log = gcs_async.read(log_filename).get_result()
if log is None:
return None
pod_re = regex.wordRE(pod)
if make_dict:
return kubelet_parser.make_dict(log.decode('utf8', 'replace'), pod_re)
else:
return log_parser.digest(log.decode('utf8', 'replace'),
error_re=pod_re, filters=filters, objref_dict=objref_dict)
def get(self):
# let's lock this down to build logs for now.
path = self.request.get('path')
if not re.match(r'^[-\w/.]+$', path):
self.abort(403)
if not path.endswith('/build-log.txt'):
self.abort(403)
content = gcs_async.read(path).get_result()
# lazy XSS prevention.
# doesn't work on terrible browsers that do content sniffing (ancient IE).
self.response.headers['Content-Type'] = 'text/plain'
self.response.write(content)
def get(self):
# let's lock this down to build logs for now.
path = self.request.get('path')
if not re.match(r'^[-\w/.]+$', path):
self.abort(403)
if not path.endswith('/build-log.txt'):
self.abort(403)
content = gcs_async.read(path).get_result()
# lazy XSS prevention.
# doesn't work on terrible browsers that do content sniffing (ancient IE).
self.response.headers['Content-Type'] = 'text/plain'
self.response.write(content)
def build_details(build_dir):
"""
Collect information from a build directory.
Args:
build_dir: GCS path containing a build's results.
Returns:
started: value from started.json {'version': ..., 'timestamp': ...}
finished: value from finished.json {'timestamp': ..., 'result': ...}
failures: list of (name, duration, text) tuples
build_log: a hilighted portion of errors in the build log. May be None.
"""
started_fut = gcs_async.read(build_dir + '/started.json')
finished = gcs_async.read(build_dir + '/finished.json').get_result()
started = started_fut.get_result()
if finished and not started:
started = 'null'
if started and not finished:
finished = 'null'
elif not (started and finished):
return
started = json.loads(started)
finished = json.loads(finished)
failures = []
junit_paths = [f.filename for f in view_base.gcs_ls('%s/artifacts' % build_dir)
if re.match(r'junit_.*\.xml', os.path.basename(f.filename))]
junit_futures = {}
for f in junit_paths:
junit_futures[gcs_async.read(f)] = f
for future in junit_futures:
junit = future.get_result()
if not junit:
continue
failures.extend(parse_junit(junit, junit_futures[future]))
failures.sort()
return started, finished, failures
def pr_builds(path, pr):
"""
Get information for all builds run by a PR.
Args:
pr: the PR number
Returns:
A dictionary of {job: [(build_number, started_json, finished.json)]}
"""
jobs_dirs_fut = gcs_async.listdirs('%s/%s%s' % (PR_PREFIX, path, pr))
print '%s/%s%s' % (PR_PREFIX, path, pr)
def base(path):
return os.path.basename(os.path.dirname(path))
jobs_futures = [(job, gcs_async.listdirs(job)) for job in jobs_dirs_fut.get_result()]
futures = []
for job, builds_fut in jobs_futures:
for build in builds_fut.get_result():
futures.append([
base(job),
base(build),
gcs_async.read('/%sstarted.json' % build),
gcs_async.read('/%sfinished.json' % build)])
futures.sort(key=lambda (job, build, s, f): (job, view_base.pad_numbers(build)), reverse=True)
jobs = {}
for job, build, started_fut, finished_fut in futures:
started = started_fut.get_result()
finished = finished_fut.get_result()
if started is not None:
started = json.loads(started)
if finished is not None:
finished = json.loads(finished)
jobs.setdefault(job, []).append((build, started, finished))
return jobs
def parse_log_file(log_filename, pod, filters=None, make_dict=False, objref_dict=None):
"""Based on make_dict, either returns the objref_dict or the parsed log file"""
log = gcs_async.read(log_filename).get_result()
if log is None:
return {}, False if make_dict else None
if pod:
bold_re = regex.wordRE(pod)
else:
bold_re = regex.error_re
if objref_dict is None:
objref_dict = {}
if make_dict and pod:
return kubelet_parser.make_dict(log.decode('utf8', 'replace'), bold_re, objref_dict)
else:
return log_parser.digest(log.decode('utf8', 'replace'),
error_re=bold_re, filters=filters, objref_dict=objref_dict)
def parse_kubelet(pod, junit, build_dir, filters):
junit_file = junit + ".xml"
tmps = [f.filename for f in gcs_ls('%s/artifacts' % build_dir)
if re.match(r'.*/tmp-node.*', f.filename)]
junit_regex = r".*" + junit_file + r".*"
kubelet_filename = ""
for folder in tmps:
tmp_contents = [f.filename for f in gcs_ls(folder)]
for f in tmp_contents:
if re.match(junit_regex, f):
for file in tmp_contents:
if re.match(r'.*kubelet\.log', file):
kubelet_filename = file
if kubelet_filename == "":
return False
kubelet_log = gcs_async.read(kubelet_filename).get_result()
if kubelet_log:
pod_re = regex.wordRE(pod)
kubelet_log = log_parser.digest(kubelet_log.decode('utf8',
'replace'), error_re=pod_re, filters=filters)
return kubelet_log
def test_read(self):
write('/foo/bar', 'test data')
self.assertEqual(gcs_async.read('/foo/bar').get_result(), 'test data')
self.assertEqual(gcs_async.read('/foo/quux').get_result(), None)
def get_build_log(build_dir):
build_log = gcs_async.read(build_dir + '/build-log.txt').get_result()
if build_log:
return log_parser.digest(build_log)
def get(self, prefix, job, build):
# pylint: disable=too-many-locals
if prefix.endswith('/directory'):
# redirect directory requests
link = gcs_async.read('/%s/%s/%s.txt' % (prefix, job, build)).get_result()
if link and link.startswith('gs://'):
self.redirect('/build/' + link.replace('gs://', ''))
return
job_dir = '/%s/%s/' % (prefix, job)
testgrid_query = testgrid.path_to_query(job_dir)
build_dir = job_dir + build
started, finished, metadata, results = build_details(build_dir)
if started is None and finished is None:
logging.warning('unable to load %s', build_dir)
self.render(
'build_404.html',
dict(build_dir=build_dir, job_dir=job_dir, job=job, build=build))
self.response.set_status(404)
return
want_build_log = False
build_log = ''
build_log_src = None
if 'log' in self.request.params or (not finished) or \
(finished and finished.get('result') != 'SUCCESS' and len(results['failed']) <= 1):
want_build_log = True
build_log = get_build_log(build_dir)
pr, pr_path, pr_digest = None, None, None
repo = '%s/%s' % (self.app.config['default_org'],
self.app.config['default_repo'])
spyglass_link = ''
external_config = get_build_config(prefix, self.app.config)
if external_config is not None:
if external_config.get('spyglass'):
spyglass_link = 'https://' + external_config['prow_url'] + '/view/gcs/' + build_dir
if '/pull/' in prefix:
pr, pr_path, pr_digest, repo = get_pr_info(prefix, self.app.config)
if want_build_log and not build_log:
build_log, build_log_src = get_running_build_log(job, build,
external_config["prow_url"])
# 'version' might be in either started or finished.
# prefer finished.
if finished and 'version' in finished:
version = finished['version']
else:
version = started and started.get('version')
commit = version and version.split('+')[-1]
issues = list(models.GHIssueDigest.find_xrefs(build_dir))
# openshift does not have a pull string because the entrypoint does not
# set it into started
ref_string = ""
if 'repos' in metadata and metadata['repos']:
if repo in metadata['repos']:
ref_string = metadata['repos'][repo]
del metadata['repos'][repo]
if len(ref_string) == 0 and started and 'pull' in started:
ref_string = started['pull']
refs = []
if len(ref_string) > 0:
for ref in ref_string.split(','):
x = ref.split(':', 1)
if len(x) == 2:
refs.append((x[0], x[1]))
else:
refs.append((x[0], ''))
work_namespace = ""
if 'work-namespace' in metadata:
work_namespace = metadata['work-namespace']
del metadata['work-namespace']
self.render('build.html', dict(
job_dir=job_dir, build_dir=build_dir, job=job, build=build,
commit=commit, started=started, finished=finished, metadata=metadata,
res=results, refs=refs,
work_namespace=work_namespace,
build_log=build_log, build_log_src=build_log_src,
issues=issues, repo=repo,
pr_path=pr_path, pr=pr, pr_digest=pr_digest,
testgrid_query=testgrid_query))
def get(self, prefix, job, build):
# pylint: disable=too-many-locals
if prefix.endswith('/directory'):
# redirect directory requests
link = gcs_async.read('/%s/%s/%s.txt' % (prefix, job, build)).get_result()
if link and link.startswith('gs://'):
self.redirect('/build/' + link.replace('gs://', ''))
return
job_dir = '/%s/%s/' % (prefix, job)
testgrid_query = testgrid.path_to_query(job_dir)
build_dir = job_dir + build
started, finished, results = build_details(build_dir)
if started is None and finished is None:
logging.warning('unable to load %s', build_dir)
self.render(
'build_404.html',
dict(build_dir=build_dir, job_dir=job_dir, job=job, build=build))
self.response.set_status(404)
return
want_build_log = False
build_log = ''
build_log_src = None
if 'log' in self.request.params or (not finished) or \
(finished and finished.get('result') != 'SUCCESS' and len(results['failed']) <= 1):
want_build_log = True
build_log = get_build_log(build_dir)
pr, pr_path, pr_digest = None, None, None
repo = '%s/%s' % (self.app.config['default_org'],
self.app.config['default_repo'])
external_config = get_build_config(prefix, self.app.config)
if external_config is not None:
if '/pull/' in prefix:
pr, pr_path, pr_digest, repo = get_pr_info(prefix, self.app.config)
if want_build_log and not build_log:
build_log, build_log_src = get_running_build_log(job, build,
external_config["prow_url"])
# 'version' might be in either started or finished.
# prefer finished.
version = finished and finished.get('version') or started and started.get('version')
commit = version and version.split('+')[-1]
issues = list(models.GHIssueDigest.find_xrefs(build_dir))
refs = []
if started and started.get('pull'):
for ref in started['pull'].split(','):
x = ref.split(':', 1)
if len(x) == 2:
refs.append((x[0], x[1]))
else:
refs.append((x[0], ''))
self.render('build.html', dict(
job_dir=job_dir, build_dir=build_dir, job=job, build=build,
commit=commit, started=started, finished=finished,
res=results, refs=refs,
build_log=build_log, build_log_src=build_log_src,
issues=issues, repo=repo,
pr_path=pr_path, pr=pr, pr_digest=pr_digest,
testgrid_query=testgrid_query))
def get(self, prefix, job, build):
# pylint: disable=too-many-locals
if prefix.endswith('/directory'):
# redirect directory requests
link = gcs_async.read('/%s/%s/%s.txt' %
(prefix, job, build)).get_result()
if link and link.startswith('gs://'):
self.redirect('/build/' + link.replace('gs://', ''))
return
job_dir = '/%s/%s/' % (prefix, job)
testgrid_query = testgrid.path_to_query(job_dir)
build_dir = job_dir + build
started, finished, results = build_details(build_dir)
if started is None and finished is None:
logging.warning('unable to load %s', build_dir)
self.render(
'build_404.html',
dict(build_dir=build_dir,
job_dir=job_dir,
job=job,
build=build))
self.response.set_status(404)
return
want_build_log = False
build_log = ''
build_log_src = None
if 'log' in self.request.params or (not finished) or \
(finished and finished.get('result') != 'SUCCESS' and len(results['failed']) <= 1):
want_build_log = True
build_log = get_build_log(build_dir)
pr, pr_path, pr_digest = None, None, None
repo = '%s/%s' % (self.app.config['default_org'],
self.app.config['default_repo'])
external_config = get_build_config(prefix, self.app.config)
if external_config is not None:
if '/pull/' in prefix:
pr, pr_path, pr_digest, repo = get_pr_info(
prefix, self.app.config)
if want_build_log and not build_log:
build_log, build_log_src = get_running_build_log(
job, build, external_config["prow_url"])
# 'version' might be in either started or finished.
# prefer finished.
version = finished and finished.get(
'version') or started and started.get('version')
commit = version and version.split('+')[-1]
issues = list(models.GHIssueDigest.find_xrefs(build_dir))
refs = []
if started and started.get('pull'):
for ref in started['pull'].split(','):
x = ref.split(':', 1)
if len(x) == 2:
refs.append((x[0], x[1]))
else:
refs.append((x[0], ''))
self.render(
'build.html',
dict(job_dir=job_dir,
build_dir=build_dir,
job=job,
build=build,
commit=commit,
started=started,
finished=finished,
res=results,
refs=refs,
build_log=build_log,
build_log_src=build_log_src,
issues=issues,
repo=repo,
pr_path=pr_path,
pr=pr,
pr_digest=pr_digest,
testgrid_query=testgrid_query))
def get_build_log(build_dir):
build_log = gcs_async.read(build_dir + '/build-log.txt').get_result()
if build_log:
return log_parser.digest(build_log)
