def print_last_command_log(self):
wide_log("")
command_output = ""
with open(self.log_path, 'r') as f:
line_number = 0
for line in f:
if line_number is not None and line_number != self.last_command_line:
line_number += 1
continue
line_number = None
command_output += line
if not self.color:
wide_log(remove_ansi_escape(command_output), end='')
else:
wide_log(command_output, end='')
wide_log("")
def print_last_command_log(self):
wide_log("")
command_output = ""
with open(self.log_path, 'r') as f:
line_number = 0
for line in f:
if line_number is not None and line_number != self.last_command_line:
line_number += 1
continue
line_number = None
command_output += line
if not self.color:
wide_log(remove_ansi_escape(command_output), end='')
else:
wide_log(command_output, end='')
wide_log("")
def command_log(self, package, msg):
if package not in self.__command_log:
raise RuntimeError("Command log received for package '{0}' before package job started: '{1}'"
.format(package, msg))
if self.__command_log[package].current_cmd is None:
raise RuntimeError("Command log received for package '{0}' before command started: '{1}'"
.format(package, msg))
self.__command_log[package].append(msg)
if not self.color:
msg = remove_ansi_escape(msg)
if not self.quiet and self.interleave:
msg = msg.rstrip(ansi('reset'))
msg = msg.rstrip()
if self.interleave and self.prefix_output:
wide_log(clr("[{package}] {msg}").format(**locals()))
else:
wide_log(msg)
def command_log(self, package, msg):
if package not in self.__command_log:
raise RuntimeError(
"Command log received for package '{0}' before package job started: '{1}'"
.format(package, msg))
if self.__command_log[package].current_cmd is None:
raise RuntimeError(
"Command log received for package '{0}' before command started: '{1}'"
.format(package, msg))
self.__command_log[package].append(msg)
if not self.color:
msg = remove_ansi_escape(msg)
if not self.quiet and self.interleave:
msg = msg.rstrip(ansi('reset'))
msg = msg.rstrip()
if self.interleave and self.prefix_output:
wide_log(clr("[{package}] {msg}").format(**locals()))
else:
wide_log(msg)
def print_exec_summary(self, completed_jobs, warned_jobs, failed_jobs):
"""
Print verbose execution summary.
"""
# Calculate the longest jid
max_jid_len = max([len(jid) for jid in self.available_jobs])
templates = {
'successful':
clr(" [@!@{gf}Successful@|] @{cf}{jid:<%d}@|" % max_jid_len),
'warned':
clr(" [ @!@{yf}Warned@|] @{cf}{jid:<%d}@|" % max_jid_len),
'failed':
clr(" [ @!@{rf}Failed@|] @{cf}{jid:<%d}@|" % max_jid_len),
'ignored':
clr(" [ @!@{kf}Ignored@|] @{cf}{jid:<%d}@|" % max_jid_len),
'abandoned':
clr(" [ @!@{rf}Abandoned@|] @{cf}{jid:<%d}@|" % max_jid_len),
}
# Calculate the maximum _printed_ length for each template
max_column_len = max([
len(remove_ansi_escape(t.format(jid=("?" * max_jid_len))))
for t in templates.values()
])
# Calculate the number of columns
number_of_columns = int((terminal_width() / max_column_len) or 1)
# Construct different categories of jobs (jid -> output template)
successfuls = {}
warneds = {}
faileds = {}
ignoreds = {}
abandoneds = {}
non_whitelisted = {}
blacklisted = {}
# Give each package an output template to use
for jid in self.available_jobs:
if jid in self.blacklisted_jobs:
blacklisted[jid] = templates['ignored']
elif jid not in self.jobs:
ignoreds[jid] = templates['ignored']
elif len(self.whitelisted_jobs
) > 0 and jid not in self.whitelisted_jobs:
non_whitelisted[jid] = templates['ignored']
elif jid in completed_jobs:
if jid in failed_jobs:
faileds[jid] = templates['failed']
elif jid in warned_jobs:
warneds[jid] = templates['warned']
else:
successfuls[jid] = templates['successful']
else:
abandoneds[jid] = templates['abandoned']
# Combine successfuls and ignoreds, sort by key
if len(successfuls) + len(ignoreds) > 0:
wide_log("")
wide_log(
clr("[{}] Successful {}:").format(self.label, self.jobs_label))
wide_log("")
print_items_in_columns(
sorted(itertools.chain(successfuls.items(), ignoreds.items())),
number_of_columns)
else:
wide_log("")
wide_log(
clr("[{}] No {} succeeded.").format(self.label,
self.jobs_label))
wide_log("")
# Print out whitelisted jobs
if len(non_whitelisted) > 0:
wide_log("")
wide_log(
clr("[{}] Non-whitelisted {}:").format(self.label,
self.jobs_label))
wide_log("")
print_items_in_columns(sorted(non_whitelisted.items()),
number_of_columns)
# Print out blacklisted jobs
if len(blacklisted) > 0:
wide_log("")
wide_log(
clr("[{}] Blacklisted {}:").format(self.label,
self.jobs_label))
wide_log("")
print_items_in_columns(sorted(blacklisted.items()),
number_of_columns)
# Print out jobs that failed
if len(faileds) > 0:
wide_log("")
wide_log(
clr("[{}] Failed {}:").format(self.label, self.jobs_label))
wide_log("")
print_items_in_columns(sorted(faileds.items()), number_of_columns)
# Print out jobs that were abandoned
if len(abandoneds) > 0:
wide_log("")
wide_log(
clr("[{}] Abandoned {}:").format(self.label, self.jobs_label))
wide_log("")
print_items_in_columns(sorted(abandoneds.items()),
number_of_columns)
wide_log("")
def print_exec_summary(self, completed_jobs, warned_jobs, failed_jobs):
"""
Print verbose execution summary.
"""
# Calculate the longest jid
max_jid_len = max([len(jid) for jid in self.available_jobs])
templates = {
'successful': clr(" [@!@{gf}Successful@|] @{cf}{jid:<%d}@|" % max_jid_len),
'warned': clr(" [ @!@{yf}Warned@|] @{cf}{jid:<%d}@|" % max_jid_len),
'failed': clr(" [ @!@{rf}Failed@|] @{cf}{jid:<%d}@|" % max_jid_len),
'ignored': clr(" [ @!@{kf}Ignored@|] @{cf}{jid:<%d}@|" % max_jid_len),
'abandoned': clr(" [ @!@{rf}Abandoned@|] @{cf}{jid:<%d}@|" % max_jid_len),
}
# Calculate the maximum _printed_ length for each template
max_column_len = max([
len(remove_ansi_escape(t.format(jid=("?" * max_jid_len))))
for t in templates.values()
])
# Calculate the number of columns
number_of_columns = (terminal_width() / max_column_len) or 1
# Construct different categories of jobs (jid -> output template)
successfuls = {}
warneds = {}
faileds = {}
ignoreds = {}
abandoneds = {}
non_whitelisted = {}
blacklisted = {}
# Give each package an output template to use
for jid in self.available_jobs:
if jid in self.blacklisted_jobs:
blacklisted[jid] = templates['ignored']
elif jid not in self.jobs:
ignoreds[jid] = templates['ignored']
elif len(self.whitelisted_jobs) > 0 and jid not in self.whitelisted_jobs:
non_whitelisted[jid] = templates['ignored']
elif jid in completed_jobs:
if jid in failed_jobs:
faileds[jid] = templates['failed']
elif jid in warned_jobs:
warneds[jid] = templates['warned']
else:
successfuls[jid] = templates['successful']
else:
abandoneds[jid] = templates['abandoned']
# Combine successfuls and ignoreds, sort by key
if len(successfuls) + len(ignoreds) > 0:
wide_log("")
wide_log(clr("[{}] Successful {}:").format(self.label, self.jobs_label))
wide_log("")
print_items_in_columns(
sorted(successfuls.items() + ignoreds.items()),
number_of_columns)
else:
wide_log("")
wide_log(clr("[{}] No {} succeeded.").format(self.label, self.jobs_label))
wide_log("")
# Print out whitelisted jobs
if len(non_whitelisted) > 0:
wide_log("")
wide_log(clr("[{}] Non-whitelisted {}:").format(self.label, self.jobs_label))
wide_log("")
print_items_in_columns(sorted(non_whitelisted.items()), number_of_columns)
# Print out blacklisted jobs
if len(blacklisted) > 0:
wide_log("")
wide_log(clr("[{}] Blacklisted {}:").format(self.label, self.jobs_label))
wide_log("")
print_items_in_columns(sorted(blacklisted.items()), number_of_columns)
# Print out jobs that failed
if len(faileds) > 0:
wide_log("")
wide_log(clr("[{}] Failed {}:").format(self.label, self.jobs_label))
wide_log("")
print_items_in_columns(sorted(faileds.items()), number_of_columns)
# Print out jobs that were abandoned
if len(abandoneds) > 0:
wide_log("")
wide_log(clr("[{}] Abandoned {}:").format(self.label, self.jobs_label))
wide_log("")
print_items_in_columns(sorted(abandoneds.items()), number_of_columns)
wide_log("")
def run(self):
try:
# Until exit
while True:
# Get a job off the queue
self.current_job = self.jobs.get()
# If the job is None, then we should shutdown
if self.current_job is None:
# Notify shutfown
self.quit()
break
# Notify that a new job was started
self.job_started(self.current_job)
# Execute each command in the job
for command in self.current_job:
install_space_locked = False
if command.lock_install_space:
self.install_space_lock.acquire()
install_space_locked = True
try:
# Log that the command being run
self.command_started(command, command.location)
# Receive lines from the running command
for line in run_command(command.cmd,
cwd=command.location):
# If it is an integer, it corresponds to the command's return code
if isinstance(line, int):
retcode = line
# If the return code is not zero
if retcode != 0:
# Log the failure (the build loop will dispatch None's)
self.command_failed(
command, command.location, retcode)
# Try to consume and throw away any and all remaining jobs in the queue
while self.jobs.get() is not None:
pass
# Once we get our None, quit
self.quit()
return
else:
self.command_finished(
command, command.location, retcode)
else:
# Otherwise it is some sort of string data
# Ensure that the data is not just ansi escape characters
if remove_ansi_escape(line).strip():
for sub_line in line.splitlines(
True): # keepends=True
if sub_line:
if command.stage_name == 'cmake':
sub_line = colorize_cmake(
sub_line)
self.command_log(sub_line)
finally:
if install_space_locked:
self.install_space_lock.release()
self.job_finished(self.current_job)
except KeyboardInterrupt:
self.quit()
except Exception as exc:
import traceback
self.quit(traceback.format_exc() + str(exc))
raise
def run(self):
try:
# Until exit
while True:
# Get a job off the queue
self.current_job = self.jobs.get()
# If the job is None, then we should shutdown
if self.current_job is None:
# Notify shutdown
self.quit()
break
# Notify that a new job was started
self.job_started(self.current_job)
# Track if the job has failed
job_has_failed = False
# Execute each command in the job
with jobserver_job():
for command in self.current_job:
install_space_locked = False
if command.lock_install_space:
self.install_space_lock.acquire()
install_space_locked = True
try:
# don't run further commands if previous one of this job failed
if job_has_failed:
break
# Log that the command being run
self.command_started(command, command.location)
# Receive lines from the running command
for line in run_command(command.cmd, cwd=command.location):
# If it is an integer, it corresponds to the command's return code
if isinstance(line, int):
retcode = line
# If the return code is not zero
if retcode != 0:
# Log the failure (the build loop will dispatch None's)
self.command_failed(command, command.location, retcode)
job_has_failed = True
break
else:
self.command_finished(command, command.location, retcode)
else:
# Otherwise it is some sort of string data
# Ensure that the data is not just ansi escape characters
if remove_ansi_escape(line).strip():
for sub_line in line.splitlines(True): # keepends=True
if sub_line:
if command.stage_name == 'cmake':
sub_line = colorize_cmake(sub_line)
self.command_log(sub_line)
finally:
if install_space_locked:
self.install_space_lock.release()
# Check if the job has failed
if job_has_failed:
self.job_failed(self.current_job)
if self.shutdown_on_failure:
# Try to consume and throw away any and all remaining jobs in the queue
while self.jobs.get() is not None:
pass
# Once we get our None, quit
self.quit()
return
else:
self.job_finished(self.current_job)
except KeyboardInterrupt:
self.quit()
except Exception as exc:
import traceback
self.quit(traceback.format_exc() + str(exc))
raise
def print_build_summary(context, packages_to_be_built, completed_packages, failed_packages):
# Calculate the longest package name
max_name_len = max([len(pkg.name) for _, pkg in context.packages])
def get_template(template_name, column_width):
templates = {
'successful': " @!@{gf}Successful@| @{cf}{package:<" + str(column_width) + "}@|",
'failed': " @!@{rf}Failed@| @{cf}{package:<" + str(column_width) + "}@|",
'not_built': " @!@{kf}Not built@| @{cf}{package:<" + str(column_width) + "}@|",
}
return templates[template_name]
# Setup templates for comparison
successful_template = get_template('successful', max_name_len)
failed_template = get_template('failed', max_name_len)
not_built_template = get_template('not_built', max_name_len)
# Calculate the maximum _printed_ length for each template
faux_package_name = ("x" * max_name_len)
templates = [
remove_ansi_escape(clr(successful_template).format(package=faux_package_name)),
remove_ansi_escape(clr(failed_template).format(package=faux_package_name)),
remove_ansi_escape(clr(not_built_template).format(package=faux_package_name)),
]
# Calculate the longest column using the longest template
max_column_len = max([len(template) for template in templates])
# Calculate the number of columns
number_of_columns = (terminal_width() / max_column_len) or 1
successfuls = {}
faileds = {}
not_builts = {}
non_whitelisted = {}
blacklisted = {}
for (_, pkg) in context.packages:
if pkg.name in context.blacklist:
blacklisted[pkg.name] = clr(not_built_template).format(package=pkg.name)
elif len(context.whitelist) > 0 and pkg.name not in context.whitelist:
non_whitelisted[pkg.name] = clr(not_built_template).format(package=pkg.name)
elif pkg.name in completed_packages:
successfuls[pkg.name] = clr(successful_template).format(package=pkg.name)
else:
if pkg.name in failed_packages:
faileds[pkg.name] = clr(failed_template).format(package=pkg.name)
else:
not_builts[pkg.name] = clr(not_built_template).format(package=pkg.name)
# Combine successfuls and not_builts, sort by key, only take values
wide_log("")
wide_log("Build summary:")
combined = dict(successfuls)
combined.update(not_builts)
non_failed = [v for k, v in sorted(combined.items(), key=operator.itemgetter(0))]
print_items_in_columns(non_failed, number_of_columns)
# Print out whitelisted packages
if len(non_whitelisted) > 0:
wide_log("")
wide_log("Non-Whitelisted Packages:")
non_whitelisted_list = [v for k, v in sorted(non_whitelisted.items(), key=operator.itemgetter(0))]
print_items_in_columns(non_whitelisted_list, number_of_columns)
# Print out blacklisted packages
if len(blacklisted) > 0:
wide_log("")
wide_log("Blacklisted Packages:")
blacklisted_list = [v for k, v in sorted(blacklisted.items(), key=operator.itemgetter(0))]
print_items_in_columns(blacklisted_list, number_of_columns)
# Faileds only, sort by key, only take values
failed = [v for k, v in sorted(faileds.items(), key=operator.itemgetter(0))]
if len(failed) > 0:
wide_log("")
wide_log("Failed packages:")
print_items_in_columns(failed, number_of_columns)
else:
wide_log("")
wide_log("All packages built successfully.")
wide_log("")
wide_log(clr("[build] @!@{gf}Successfully@| built '@!@{cf}{0}@|' packages, "
"@!@{rf}failed@| to build '@!@{cf}{1}@|' packages, "
"and @!@{kf}did not try to build@| '@!@{cf}{2}@|' packages.").format(
len(successfuls), len(faileds), len(not_builts)
))