Exemplo n.º 1
0
 def print_last_command_log(self):
     wide_log("")
     command_output = ""
     with open(self.log_path, 'r') as f:
         line_number = 0
         for line in f:
             if line_number is not None and line_number != self.last_command_line:
                 line_number += 1
                 continue
             line_number = None
             command_output += line
     if not self.color:
         wide_log(remove_ansi_escape(command_output), end='')
     else:
         wide_log(command_output, end='')
     wide_log("")
Exemplo n.º 2
0
 def print_last_command_log(self):
     wide_log("")
     command_output = ""
     with open(self.log_path, 'r') as f:
         line_number = 0
         for line in f:
             if line_number is not None and line_number != self.last_command_line:
                 line_number += 1
                 continue
             line_number = None
             command_output += line
     if not self.color:
         wide_log(remove_ansi_escape(command_output), end='')
     else:
         wide_log(command_output, end='')
     wide_log("")
Exemplo n.º 3
0
 def command_log(self, package, msg):
     if package not in self.__command_log:
         raise RuntimeError("Command log received for package '{0}' before package job started: '{1}'"
                            .format(package, msg))
     if self.__command_log[package].current_cmd is None:
         raise RuntimeError("Command log received for package '{0}' before command started: '{1}'"
                            .format(package, msg))
     self.__command_log[package].append(msg)
     if not self.color:
         msg = remove_ansi_escape(msg)
     if not self.quiet and self.interleave:
         msg = msg.rstrip(ansi('reset'))
         msg = msg.rstrip()
         if self.interleave and self.prefix_output:
             wide_log(clr("[{package}] {msg}").format(**locals()))
         else:
             wide_log(msg)
Exemplo n.º 4
0
 def command_log(self, package, msg):
     if package not in self.__command_log:
         raise RuntimeError(
             "Command log received for package '{0}' before package job started: '{1}'"
             .format(package, msg))
     if self.__command_log[package].current_cmd is None:
         raise RuntimeError(
             "Command log received for package '{0}' before command started: '{1}'"
             .format(package, msg))
     self.__command_log[package].append(msg)
     if not self.color:
         msg = remove_ansi_escape(msg)
     if not self.quiet and self.interleave:
         msg = msg.rstrip(ansi('reset'))
         msg = msg.rstrip()
         if self.interleave and self.prefix_output:
             wide_log(clr("[{package}] {msg}").format(**locals()))
         else:
             wide_log(msg)
Exemplo n.º 5
0
    def print_exec_summary(self, completed_jobs, warned_jobs, failed_jobs):
        """
        Print verbose execution summary.
        """

        # Calculate the longest jid
        max_jid_len = max([len(jid) for jid in self.available_jobs])

        templates = {
            'successful':
            clr(" [@!@{gf}Successful@|] @{cf}{jid:<%d}@|" % max_jid_len),
            'warned':
            clr(" [    @!@{yf}Warned@|] @{cf}{jid:<%d}@|" % max_jid_len),
            'failed':
            clr(" [    @!@{rf}Failed@|] @{cf}{jid:<%d}@|" % max_jid_len),
            'ignored':
            clr(" [   @!@{kf}Ignored@|] @{cf}{jid:<%d}@|" % max_jid_len),
            'abandoned':
            clr(" [ @!@{rf}Abandoned@|] @{cf}{jid:<%d}@|" % max_jid_len),
        }

        # Calculate the maximum _printed_ length for each template
        max_column_len = max([
            len(remove_ansi_escape(t.format(jid=("?" * max_jid_len))))
            for t in templates.values()
        ])

        # Calculate the number of columns
        number_of_columns = int((terminal_width() / max_column_len) or 1)

        # Construct different categories of jobs (jid -> output template)
        successfuls = {}
        warneds = {}
        faileds = {}
        ignoreds = {}
        abandoneds = {}
        non_whitelisted = {}
        blacklisted = {}

        # Give each package an output template to use
        for jid in self.available_jobs:
            if jid in self.blacklisted_jobs:
                blacklisted[jid] = templates['ignored']
            elif jid not in self.jobs:
                ignoreds[jid] = templates['ignored']
            elif len(self.whitelisted_jobs
                     ) > 0 and jid not in self.whitelisted_jobs:
                non_whitelisted[jid] = templates['ignored']
            elif jid in completed_jobs:
                if jid in failed_jobs:
                    faileds[jid] = templates['failed']
                elif jid in warned_jobs:
                    warneds[jid] = templates['warned']
                else:
                    successfuls[jid] = templates['successful']
            else:
                abandoneds[jid] = templates['abandoned']

        # Combine successfuls and ignoreds, sort by key
        if len(successfuls) + len(ignoreds) > 0:
            wide_log("")
            wide_log(
                clr("[{}] Successful {}:").format(self.label, self.jobs_label))
            wide_log("")
            print_items_in_columns(
                sorted(itertools.chain(successfuls.items(), ignoreds.items())),
                number_of_columns)
        else:
            wide_log("")
            wide_log(
                clr("[{}] No {} succeeded.").format(self.label,
                                                    self.jobs_label))
            wide_log("")

        # Print out whitelisted jobs
        if len(non_whitelisted) > 0:
            wide_log("")
            wide_log(
                clr("[{}] Non-whitelisted {}:").format(self.label,
                                                       self.jobs_label))
            wide_log("")
            print_items_in_columns(sorted(non_whitelisted.items()),
                                   number_of_columns)

        # Print out blacklisted jobs
        if len(blacklisted) > 0:
            wide_log("")
            wide_log(
                clr("[{}] Blacklisted {}:").format(self.label,
                                                   self.jobs_label))
            wide_log("")
            print_items_in_columns(sorted(blacklisted.items()),
                                   number_of_columns)

        # Print out jobs that failed
        if len(faileds) > 0:
            wide_log("")
            wide_log(
                clr("[{}] Failed {}:").format(self.label, self.jobs_label))
            wide_log("")
            print_items_in_columns(sorted(faileds.items()), number_of_columns)

        # Print out jobs that were abandoned
        if len(abandoneds) > 0:
            wide_log("")
            wide_log(
                clr("[{}] Abandoned {}:").format(self.label, self.jobs_label))
            wide_log("")
            print_items_in_columns(sorted(abandoneds.items()),
                                   number_of_columns)

        wide_log("")
Exemplo n.º 6
0
    def print_exec_summary(self, completed_jobs, warned_jobs, failed_jobs):
        """
        Print verbose execution summary.
        """

        # Calculate the longest jid
        max_jid_len = max([len(jid) for jid in self.available_jobs])

        templates = {
            'successful': clr(" [@!@{gf}Successful@|] @{cf}{jid:<%d}@|" % max_jid_len),
            'warned': clr(" [    @!@{yf}Warned@|] @{cf}{jid:<%d}@|" % max_jid_len),
            'failed': clr(" [    @!@{rf}Failed@|] @{cf}{jid:<%d}@|" % max_jid_len),
            'ignored': clr(" [   @!@{kf}Ignored@|] @{cf}{jid:<%d}@|" % max_jid_len),
            'abandoned': clr(" [ @!@{rf}Abandoned@|] @{cf}{jid:<%d}@|" % max_jid_len),
        }

        # Calculate the maximum _printed_ length for each template
        max_column_len = max([
            len(remove_ansi_escape(t.format(jid=("?" * max_jid_len))))
            for t in templates.values()
        ])

        # Calculate the number of columns
        number_of_columns = (terminal_width() / max_column_len) or 1

        # Construct different categories of jobs (jid -> output template)
        successfuls = {}
        warneds = {}
        faileds = {}
        ignoreds = {}
        abandoneds = {}
        non_whitelisted = {}
        blacklisted = {}

        # Give each package an output template to use
        for jid in self.available_jobs:
            if jid in self.blacklisted_jobs:
                blacklisted[jid] = templates['ignored']
            elif jid not in self.jobs:
                ignoreds[jid] = templates['ignored']
            elif len(self.whitelisted_jobs) > 0 and jid not in self.whitelisted_jobs:
                non_whitelisted[jid] = templates['ignored']
            elif jid in completed_jobs:
                if jid in failed_jobs:
                    faileds[jid] = templates['failed']
                elif jid in warned_jobs:
                    warneds[jid] = templates['warned']
                else:
                    successfuls[jid] = templates['successful']
            else:
                abandoneds[jid] = templates['abandoned']

        # Combine successfuls and ignoreds, sort by key
        if len(successfuls) + len(ignoreds) > 0:
            wide_log("")
            wide_log(clr("[{}] Successful {}:").format(self.label, self.jobs_label))
            wide_log("")
            print_items_in_columns(
                sorted(successfuls.items() + ignoreds.items()),
                number_of_columns)
        else:
            wide_log("")
            wide_log(clr("[{}] No {} succeeded.").format(self.label, self.jobs_label))
            wide_log("")

        # Print out whitelisted jobs
        if len(non_whitelisted) > 0:
            wide_log("")
            wide_log(clr("[{}] Non-whitelisted {}:").format(self.label, self.jobs_label))
            wide_log("")
            print_items_in_columns(sorted(non_whitelisted.items()), number_of_columns)

        # Print out blacklisted jobs
        if len(blacklisted) > 0:
            wide_log("")
            wide_log(clr("[{}] Blacklisted {}:").format(self.label, self.jobs_label))
            wide_log("")
            print_items_in_columns(sorted(blacklisted.items()), number_of_columns)

        # Print out jobs that failed
        if len(faileds) > 0:
            wide_log("")
            wide_log(clr("[{}] Failed {}:").format(self.label, self.jobs_label))
            wide_log("")
            print_items_in_columns(sorted(faileds.items()), number_of_columns)

        # Print out jobs that were abandoned
        if len(abandoneds) > 0:
            wide_log("")
            wide_log(clr("[{}] Abandoned {}:").format(self.label, self.jobs_label))
            wide_log("")
            print_items_in_columns(sorted(abandoneds.items()), number_of_columns)

        wide_log("")
Exemplo n.º 7
0
 def run(self):
     try:
         # Until exit
         while True:
             # Get a job off the queue
             self.current_job = self.jobs.get()
             # If the job is None, then we should shutdown
             if self.current_job is None:
                 # Notify shutfown
                 self.quit()
                 break
             # Notify that a new job was started
             self.job_started(self.current_job)
             # Execute each command in the job
             for command in self.current_job:
                 install_space_locked = False
                 if command.lock_install_space:
                     self.install_space_lock.acquire()
                     install_space_locked = True
                 try:
                     # Log that the command being run
                     self.command_started(command, command.location)
                     # Receive lines from the running command
                     for line in run_command(command.cmd,
                                             cwd=command.location):
                         # If it is an integer, it corresponds to the command's return code
                         if isinstance(line, int):
                             retcode = line
                             # If the return code is not zero
                             if retcode != 0:
                                 # Log the failure (the build loop will dispatch None's)
                                 self.command_failed(
                                     command, command.location, retcode)
                                 # Try to consume and throw away any and all remaining jobs in the queue
                                 while self.jobs.get() is not None:
                                     pass
                                 # Once we get our None, quit
                                 self.quit()
                                 return
                             else:
                                 self.command_finished(
                                     command, command.location, retcode)
                         else:
                             # Otherwise it is some sort of string data
                             # Ensure that the data is not just ansi escape characters
                             if remove_ansi_escape(line).strip():
                                 for sub_line in line.splitlines(
                                         True):  # keepends=True
                                     if sub_line:
                                         if command.stage_name == 'cmake':
                                             sub_line = colorize_cmake(
                                                 sub_line)
                                         self.command_log(sub_line)
                 finally:
                     if install_space_locked:
                         self.install_space_lock.release()
             self.job_finished(self.current_job)
     except KeyboardInterrupt:
         self.quit()
     except Exception as exc:
         import traceback
         self.quit(traceback.format_exc() + str(exc))
         raise
Exemplo n.º 8
0
    def run(self):
        try:
            # Until exit
            while True:
                # Get a job off the queue
                self.current_job = self.jobs.get()
                # If the job is None, then we should shutdown
                if self.current_job is None:
                    # Notify shutdown
                    self.quit()
                    break
                # Notify that a new job was started
                self.job_started(self.current_job)

                # Track if the job has failed
                job_has_failed = False

                # Execute each command in the job
                with jobserver_job():
                    for command in self.current_job:
                        install_space_locked = False
                        if command.lock_install_space:
                            self.install_space_lock.acquire()
                            install_space_locked = True
                        try:
                            # don't run further commands if previous one of this job failed
                            if job_has_failed:
                                break
                            # Log that the command being run
                            self.command_started(command, command.location)
                            # Receive lines from the running command
                            for line in run_command(command.cmd, cwd=command.location):
                                # If it is an integer, it corresponds to the command's return code
                                if isinstance(line, int):
                                    retcode = line
                                    # If the return code is not zero
                                    if retcode != 0:
                                        # Log the failure (the build loop will dispatch None's)
                                        self.command_failed(command, command.location, retcode)
                                        job_has_failed = True
                                        break
                                    else:
                                        self.command_finished(command, command.location, retcode)
                                else:
                                    # Otherwise it is some sort of string data
                                    # Ensure that the data is not just ansi escape characters
                                    if remove_ansi_escape(line).strip():
                                        for sub_line in line.splitlines(True):  # keepends=True
                                            if sub_line:
                                                if command.stage_name == 'cmake':
                                                    sub_line = colorize_cmake(sub_line)
                                                self.command_log(sub_line)
                        finally:
                            if install_space_locked:
                                self.install_space_lock.release()

                # Check if the job has failed
                if job_has_failed:
                    self.job_failed(self.current_job)
                    if self.shutdown_on_failure:
                        # Try to consume and throw away any and all remaining jobs in the queue
                        while self.jobs.get() is not None:
                            pass
                        # Once we get our None, quit
                        self.quit()
                        return
                else:
                    self.job_finished(self.current_job)

        except KeyboardInterrupt:
            self.quit()
        except Exception as exc:
            import traceback
            self.quit(traceback.format_exc() + str(exc))
            raise
Exemplo n.º 9
0
def print_build_summary(context, packages_to_be_built, completed_packages, failed_packages):
    # Calculate the longest package name
    max_name_len = max([len(pkg.name) for _, pkg in context.packages])

    def get_template(template_name, column_width):
        templates = {
            'successful': " @!@{gf}Successful@| @{cf}{package:<" + str(column_width) + "}@|",
            'failed': " @!@{rf}Failed@|     @{cf}{package:<" + str(column_width) + "}@|",
            'not_built': " @!@{kf}Not built@|  @{cf}{package:<" + str(column_width) + "}@|",
        }
        return templates[template_name]

    # Setup templates for comparison
    successful_template = get_template('successful', max_name_len)
    failed_template = get_template('failed', max_name_len)
    not_built_template = get_template('not_built', max_name_len)
    # Calculate the maximum _printed_ length for each template
    faux_package_name = ("x" * max_name_len)
    templates = [
        remove_ansi_escape(clr(successful_template).format(package=faux_package_name)),
        remove_ansi_escape(clr(failed_template).format(package=faux_package_name)),
        remove_ansi_escape(clr(not_built_template).format(package=faux_package_name)),
    ]
    # Calculate the longest column using the longest template
    max_column_len = max([len(template) for template in templates])
    # Calculate the number of columns
    number_of_columns = (terminal_width() / max_column_len) or 1

    successfuls = {}
    faileds = {}
    not_builts = {}
    non_whitelisted = {}
    blacklisted = {}

    for (_, pkg) in context.packages:
        if pkg.name in context.blacklist:
            blacklisted[pkg.name] = clr(not_built_template).format(package=pkg.name)
        elif len(context.whitelist) > 0 and pkg.name not in context.whitelist:
            non_whitelisted[pkg.name] = clr(not_built_template).format(package=pkg.name)
        elif pkg.name in completed_packages:
            successfuls[pkg.name] = clr(successful_template).format(package=pkg.name)
        else:
            if pkg.name in failed_packages:
                faileds[pkg.name] = clr(failed_template).format(package=pkg.name)
            else:
                not_builts[pkg.name] = clr(not_built_template).format(package=pkg.name)

    # Combine successfuls and not_builts, sort by key, only take values
    wide_log("")
    wide_log("Build summary:")
    combined = dict(successfuls)
    combined.update(not_builts)
    non_failed = [v for k, v in sorted(combined.items(), key=operator.itemgetter(0))]
    print_items_in_columns(non_failed, number_of_columns)

    # Print out whitelisted packages
    if len(non_whitelisted) > 0:
        wide_log("")
        wide_log("Non-Whitelisted Packages:")
        non_whitelisted_list = [v for k, v in sorted(non_whitelisted.items(), key=operator.itemgetter(0))]
        print_items_in_columns(non_whitelisted_list, number_of_columns)

    # Print out blacklisted packages
    if len(blacklisted) > 0:
        wide_log("")
        wide_log("Blacklisted Packages:")
        blacklisted_list = [v for k, v in sorted(blacklisted.items(), key=operator.itemgetter(0))]
        print_items_in_columns(blacklisted_list, number_of_columns)

    # Faileds only, sort by key, only take values
    failed = [v for k, v in sorted(faileds.items(), key=operator.itemgetter(0))]
    if len(failed) > 0:
        wide_log("")
        wide_log("Failed packages:")
        print_items_in_columns(failed, number_of_columns)
    else:
        wide_log("")
        wide_log("All packages built successfully.")

    wide_log("")
    wide_log(clr("[build] @!@{gf}Successfully@| built '@!@{cf}{0}@|' packages, "
                 "@!@{rf}failed@| to build '@!@{cf}{1}@|' packages, "
                 "and @!@{kf}did not try to build@| '@!@{cf}{2}@|' packages.").format(
        len(successfuls), len(faileds), len(not_builts)
    ))