def print_last_command_log(self): wide_log("") command_output = "" with open(self.log_path, 'r') as f: line_number = 0 for line in f: if line_number is not None and line_number != self.last_command_line: line_number += 1 continue line_number = None command_output += line if not self.color: wide_log(remove_ansi_escape(command_output), end='') else: wide_log(command_output, end='') wide_log("")
def command_log(self, package, msg): if package not in self.__command_log: raise RuntimeError("Command log received for package '{0}' before package job started: '{1}'" .format(package, msg)) if self.__command_log[package].current_cmd is None: raise RuntimeError("Command log received for package '{0}' before command started: '{1}'" .format(package, msg)) self.__command_log[package].append(msg) if not self.color: msg = remove_ansi_escape(msg) if not self.quiet and self.interleave: msg = msg.rstrip(ansi('reset')) msg = msg.rstrip() if self.interleave and self.prefix_output: wide_log(clr("[{package}] {msg}").format(**locals())) else: wide_log(msg)
def command_log(self, package, msg): if package not in self.__command_log: raise RuntimeError( "Command log received for package '{0}' before package job started: '{1}'" .format(package, msg)) if self.__command_log[package].current_cmd is None: raise RuntimeError( "Command log received for package '{0}' before command started: '{1}'" .format(package, msg)) self.__command_log[package].append(msg) if not self.color: msg = remove_ansi_escape(msg) if not self.quiet and self.interleave: msg = msg.rstrip(ansi('reset')) msg = msg.rstrip() if self.interleave and self.prefix_output: wide_log(clr("[{package}] {msg}").format(**locals())) else: wide_log(msg)
def print_exec_summary(self, completed_jobs, warned_jobs, failed_jobs): """ Print verbose execution summary. """ # Calculate the longest jid max_jid_len = max([len(jid) for jid in self.available_jobs]) templates = { 'successful': clr(" [@!@{gf}Successful@|] @{cf}{jid:<%d}@|" % max_jid_len), 'warned': clr(" [ @!@{yf}Warned@|] @{cf}{jid:<%d}@|" % max_jid_len), 'failed': clr(" [ @!@{rf}Failed@|] @{cf}{jid:<%d}@|" % max_jid_len), 'ignored': clr(" [ @!@{kf}Ignored@|] @{cf}{jid:<%d}@|" % max_jid_len), 'abandoned': clr(" [ @!@{rf}Abandoned@|] @{cf}{jid:<%d}@|" % max_jid_len), } # Calculate the maximum _printed_ length for each template max_column_len = max([ len(remove_ansi_escape(t.format(jid=("?" * max_jid_len)))) for t in templates.values() ]) # Calculate the number of columns number_of_columns = int((terminal_width() / max_column_len) or 1) # Construct different categories of jobs (jid -> output template) successfuls = {} warneds = {} faileds = {} ignoreds = {} abandoneds = {} non_whitelisted = {} blacklisted = {} # Give each package an output template to use for jid in self.available_jobs: if jid in self.blacklisted_jobs: blacklisted[jid] = templates['ignored'] elif jid not in self.jobs: ignoreds[jid] = templates['ignored'] elif len(self.whitelisted_jobs ) > 0 and jid not in self.whitelisted_jobs: non_whitelisted[jid] = templates['ignored'] elif jid in completed_jobs: if jid in failed_jobs: faileds[jid] = templates['failed'] elif jid in warned_jobs: warneds[jid] = templates['warned'] else: successfuls[jid] = templates['successful'] else: abandoneds[jid] = templates['abandoned'] # Combine successfuls and ignoreds, sort by key if len(successfuls) + len(ignoreds) > 0: wide_log("") wide_log( clr("[{}] Successful {}:").format(self.label, self.jobs_label)) wide_log("") print_items_in_columns( sorted(itertools.chain(successfuls.items(), ignoreds.items())), number_of_columns) else: wide_log("") wide_log( clr("[{}] No {} succeeded.").format(self.label, self.jobs_label)) wide_log("") # Print out whitelisted jobs if len(non_whitelisted) > 0: wide_log("") wide_log( clr("[{}] Non-whitelisted {}:").format(self.label, self.jobs_label)) wide_log("") print_items_in_columns(sorted(non_whitelisted.items()), number_of_columns) # Print out blacklisted jobs if len(blacklisted) > 0: wide_log("") wide_log( clr("[{}] Blacklisted {}:").format(self.label, self.jobs_label)) wide_log("") print_items_in_columns(sorted(blacklisted.items()), number_of_columns) # Print out jobs that failed if len(faileds) > 0: wide_log("") wide_log( clr("[{}] Failed {}:").format(self.label, self.jobs_label)) wide_log("") print_items_in_columns(sorted(faileds.items()), number_of_columns) # Print out jobs that were abandoned if len(abandoneds) > 0: wide_log("") wide_log( clr("[{}] Abandoned {}:").format(self.label, self.jobs_label)) wide_log("") print_items_in_columns(sorted(abandoneds.items()), number_of_columns) wide_log("")
def print_exec_summary(self, completed_jobs, warned_jobs, failed_jobs): """ Print verbose execution summary. """ # Calculate the longest jid max_jid_len = max([len(jid) for jid in self.available_jobs]) templates = { 'successful': clr(" [@!@{gf}Successful@|] @{cf}{jid:<%d}@|" % max_jid_len), 'warned': clr(" [ @!@{yf}Warned@|] @{cf}{jid:<%d}@|" % max_jid_len), 'failed': clr(" [ @!@{rf}Failed@|] @{cf}{jid:<%d}@|" % max_jid_len), 'ignored': clr(" [ @!@{kf}Ignored@|] @{cf}{jid:<%d}@|" % max_jid_len), 'abandoned': clr(" [ @!@{rf}Abandoned@|] @{cf}{jid:<%d}@|" % max_jid_len), } # Calculate the maximum _printed_ length for each template max_column_len = max([ len(remove_ansi_escape(t.format(jid=("?" * max_jid_len)))) for t in templates.values() ]) # Calculate the number of columns number_of_columns = (terminal_width() / max_column_len) or 1 # Construct different categories of jobs (jid -> output template) successfuls = {} warneds = {} faileds = {} ignoreds = {} abandoneds = {} non_whitelisted = {} blacklisted = {} # Give each package an output template to use for jid in self.available_jobs: if jid in self.blacklisted_jobs: blacklisted[jid] = templates['ignored'] elif jid not in self.jobs: ignoreds[jid] = templates['ignored'] elif len(self.whitelisted_jobs) > 0 and jid not in self.whitelisted_jobs: non_whitelisted[jid] = templates['ignored'] elif jid in completed_jobs: if jid in failed_jobs: faileds[jid] = templates['failed'] elif jid in warned_jobs: warneds[jid] = templates['warned'] else: successfuls[jid] = templates['successful'] else: abandoneds[jid] = templates['abandoned'] # Combine successfuls and ignoreds, sort by key if len(successfuls) + len(ignoreds) > 0: wide_log("") wide_log(clr("[{}] Successful {}:").format(self.label, self.jobs_label)) wide_log("") print_items_in_columns( sorted(successfuls.items() + ignoreds.items()), number_of_columns) else: wide_log("") wide_log(clr("[{}] No {} succeeded.").format(self.label, self.jobs_label)) wide_log("") # Print out whitelisted jobs if len(non_whitelisted) > 0: wide_log("") wide_log(clr("[{}] Non-whitelisted {}:").format(self.label, self.jobs_label)) wide_log("") print_items_in_columns(sorted(non_whitelisted.items()), number_of_columns) # Print out blacklisted jobs if len(blacklisted) > 0: wide_log("") wide_log(clr("[{}] Blacklisted {}:").format(self.label, self.jobs_label)) wide_log("") print_items_in_columns(sorted(blacklisted.items()), number_of_columns) # Print out jobs that failed if len(faileds) > 0: wide_log("") wide_log(clr("[{}] Failed {}:").format(self.label, self.jobs_label)) wide_log("") print_items_in_columns(sorted(faileds.items()), number_of_columns) # Print out jobs that were abandoned if len(abandoneds) > 0: wide_log("") wide_log(clr("[{}] Abandoned {}:").format(self.label, self.jobs_label)) wide_log("") print_items_in_columns(sorted(abandoneds.items()), number_of_columns) wide_log("")
def run(self): try: # Until exit while True: # Get a job off the queue self.current_job = self.jobs.get() # If the job is None, then we should shutdown if self.current_job is None: # Notify shutfown self.quit() break # Notify that a new job was started self.job_started(self.current_job) # Execute each command in the job for command in self.current_job: install_space_locked = False if command.lock_install_space: self.install_space_lock.acquire() install_space_locked = True try: # Log that the command being run self.command_started(command, command.location) # Receive lines from the running command for line in run_command(command.cmd, cwd=command.location): # If it is an integer, it corresponds to the command's return code if isinstance(line, int): retcode = line # If the return code is not zero if retcode != 0: # Log the failure (the build loop will dispatch None's) self.command_failed( command, command.location, retcode) # Try to consume and throw away any and all remaining jobs in the queue while self.jobs.get() is not None: pass # Once we get our None, quit self.quit() return else: self.command_finished( command, command.location, retcode) else: # Otherwise it is some sort of string data # Ensure that the data is not just ansi escape characters if remove_ansi_escape(line).strip(): for sub_line in line.splitlines( True): # keepends=True if sub_line: if command.stage_name == 'cmake': sub_line = colorize_cmake( sub_line) self.command_log(sub_line) finally: if install_space_locked: self.install_space_lock.release() self.job_finished(self.current_job) except KeyboardInterrupt: self.quit() except Exception as exc: import traceback self.quit(traceback.format_exc() + str(exc)) raise
def run(self): try: # Until exit while True: # Get a job off the queue self.current_job = self.jobs.get() # If the job is None, then we should shutdown if self.current_job is None: # Notify shutdown self.quit() break # Notify that a new job was started self.job_started(self.current_job) # Track if the job has failed job_has_failed = False # Execute each command in the job with jobserver_job(): for command in self.current_job: install_space_locked = False if command.lock_install_space: self.install_space_lock.acquire() install_space_locked = True try: # don't run further commands if previous one of this job failed if job_has_failed: break # Log that the command being run self.command_started(command, command.location) # Receive lines from the running command for line in run_command(command.cmd, cwd=command.location): # If it is an integer, it corresponds to the command's return code if isinstance(line, int): retcode = line # If the return code is not zero if retcode != 0: # Log the failure (the build loop will dispatch None's) self.command_failed(command, command.location, retcode) job_has_failed = True break else: self.command_finished(command, command.location, retcode) else: # Otherwise it is some sort of string data # Ensure that the data is not just ansi escape characters if remove_ansi_escape(line).strip(): for sub_line in line.splitlines(True): # keepends=True if sub_line: if command.stage_name == 'cmake': sub_line = colorize_cmake(sub_line) self.command_log(sub_line) finally: if install_space_locked: self.install_space_lock.release() # Check if the job has failed if job_has_failed: self.job_failed(self.current_job) if self.shutdown_on_failure: # Try to consume and throw away any and all remaining jobs in the queue while self.jobs.get() is not None: pass # Once we get our None, quit self.quit() return else: self.job_finished(self.current_job) except KeyboardInterrupt: self.quit() except Exception as exc: import traceback self.quit(traceback.format_exc() + str(exc)) raise
def print_build_summary(context, packages_to_be_built, completed_packages, failed_packages): # Calculate the longest package name max_name_len = max([len(pkg.name) for _, pkg in context.packages]) def get_template(template_name, column_width): templates = { 'successful': " @!@{gf}Successful@| @{cf}{package:<" + str(column_width) + "}@|", 'failed': " @!@{rf}Failed@| @{cf}{package:<" + str(column_width) + "}@|", 'not_built': " @!@{kf}Not built@| @{cf}{package:<" + str(column_width) + "}@|", } return templates[template_name] # Setup templates for comparison successful_template = get_template('successful', max_name_len) failed_template = get_template('failed', max_name_len) not_built_template = get_template('not_built', max_name_len) # Calculate the maximum _printed_ length for each template faux_package_name = ("x" * max_name_len) templates = [ remove_ansi_escape(clr(successful_template).format(package=faux_package_name)), remove_ansi_escape(clr(failed_template).format(package=faux_package_name)), remove_ansi_escape(clr(not_built_template).format(package=faux_package_name)), ] # Calculate the longest column using the longest template max_column_len = max([len(template) for template in templates]) # Calculate the number of columns number_of_columns = (terminal_width() / max_column_len) or 1 successfuls = {} faileds = {} not_builts = {} non_whitelisted = {} blacklisted = {} for (_, pkg) in context.packages: if pkg.name in context.blacklist: blacklisted[pkg.name] = clr(not_built_template).format(package=pkg.name) elif len(context.whitelist) > 0 and pkg.name not in context.whitelist: non_whitelisted[pkg.name] = clr(not_built_template).format(package=pkg.name) elif pkg.name in completed_packages: successfuls[pkg.name] = clr(successful_template).format(package=pkg.name) else: if pkg.name in failed_packages: faileds[pkg.name] = clr(failed_template).format(package=pkg.name) else: not_builts[pkg.name] = clr(not_built_template).format(package=pkg.name) # Combine successfuls and not_builts, sort by key, only take values wide_log("") wide_log("Build summary:") combined = dict(successfuls) combined.update(not_builts) non_failed = [v for k, v in sorted(combined.items(), key=operator.itemgetter(0))] print_items_in_columns(non_failed, number_of_columns) # Print out whitelisted packages if len(non_whitelisted) > 0: wide_log("") wide_log("Non-Whitelisted Packages:") non_whitelisted_list = [v for k, v in sorted(non_whitelisted.items(), key=operator.itemgetter(0))] print_items_in_columns(non_whitelisted_list, number_of_columns) # Print out blacklisted packages if len(blacklisted) > 0: wide_log("") wide_log("Blacklisted Packages:") blacklisted_list = [v for k, v in sorted(blacklisted.items(), key=operator.itemgetter(0))] print_items_in_columns(blacklisted_list, number_of_columns) # Faileds only, sort by key, only take values failed = [v for k, v in sorted(faileds.items(), key=operator.itemgetter(0))] if len(failed) > 0: wide_log("") wide_log("Failed packages:") print_items_in_columns(failed, number_of_columns) else: wide_log("") wide_log("All packages built successfully.") wide_log("") wide_log(clr("[build] @!@{gf}Successfully@| built '@!@{cf}{0}@|' packages, " "@!@{rf}failed@| to build '@!@{cf}{1}@|' packages, " "and @!@{kf}did not try to build@| '@!@{cf}{2}@|' packages.").format( len(successfuls), len(faileds), len(not_builts) ))