Ejemplo n.º 1
0
    def state_iterator(self, buffer):
        line = None
        new_tests = []
        job_count, boot_count = 0, 0
        min_stack_size = 0
        stack = status_lib.status_stack()
        current_kernel = kernel("", [])  # UNKNOWN
        current_status = status_lib.statuses[-1]
        current_reason = None
        started_time_stack = [None]
        subdir_stack = [None]
        running_test = None
        running_reasons = set()
        yield []  # we're ready to start running

        # create a RUNNING SERVER_JOB entry to represent the entire test
        running_job = test.parse_partial_test(self.job, "----", "SERVER_JOB",
                                              "", current_kernel,
                                              self.job.started_time)
        new_tests.append(running_job)

        while True:
            # are we finished with parsing?
            if buffer.size() == 0 and self.finished:
                if stack.size() == 0:
                    break
                # we have status lines left on the stack,
                # we need to implicitly abort them first
                logging.debug('Unexpected end of job, aborting')
                abort_subdir_stack = list(subdir_stack)
                if self.job.aborted_by:
                    reason = "Job aborted by %s" % self.job.aborted_by
                    reason += self.job.aborted_on.strftime(
                        " at %b %d %H:%M:%S")
                else:
                    reason = "Job aborted unexpectedly"

                timestamp = line.optional_fields.get('timestamp')
                for i in reversed(xrange(stack.size())):
                    if abort_subdir_stack:
                        subdir = abort_subdir_stack.pop()
                    else:
                        subdir = None
                    abort = self.make_dummy_abort(i, subdir, subdir, timestamp,
                                                  reason)
                    buffer.put(abort)

            # stop processing once the buffer is empty
            if buffer.size() == 0:
                yield new_tests
                new_tests = []
                continue

            # reinitialize the per-iteration state
            started_time = None
            finished_time = None

            # get the next line
            raw_line = status_lib.clean_raw_line(buffer.get())
            logging.debug('STATUS: %s', raw_line.strip())
            line = status_line.parse_line(raw_line)
            if line is None:
                logging.debug('non-status line, ignoring')
                continue

            # do an initial sanity check of the indentation
            expected_indent = stack.size()
            if line.type == "END":
                expected_indent -= 1
            if line.indent < expected_indent:
                # ABORT the current level if indentation was unexpectedly low
                self.put_back_line_and_abort(
                    buffer, raw_line,
                    stack.size() - 1, subdir_stack[-1],
                    line.optional_fields.get("timestamp"), line.reason)
                continue
            elif line.indent > expected_indent:
                # ignore the log if the indent was unexpectedly high
                logging.debug("unexpected extra indentation, ignoring")
                continue

            # initial line processing
            if line.type == "START":
                stack.start()
                started_time = line.get_timestamp()
                if (line.testname is None and line.subdir is None
                        and not running_test):
                    # we just started a client, all tests are relative to here
                    min_stack_size = stack.size()
                    # start a "RUNNING" CLIENT_JOB entry
                    job_name = "CLIENT_JOB.%d" % job_count
                    running_client = test.parse_partial_test(
                        self.job, None, job_name, "", current_kernel,
                        started_time)
                    logging.debug("RUNNING: %s", running_client.status)
                    logging.debug("Testname: %s", running_client.testname)
                    new_tests.append(running_client)
                elif stack.size() == min_stack_size + 1 and not running_test:
                    # we just started a new test, insert a running record
                    running_reasons = set()
                    if line.reason:
                        running_reasons.add(line.reason)
                    running_test = test.parse_partial_test(
                        self.job, line.subdir, line.testname, line.reason,
                        current_kernel, started_time)
                    logging.debug("RUNNING: %s", running_test.status)
                    logging.debug("Subdir: %s", running_test.subdir)
                    logging.debug("Testname: %s", running_test.testname)
                    logging.debug("Reason: %s", running_test.reason)
                    new_tests.append(running_test)
                started_time_stack.append(started_time)
                subdir_stack.append(line.subdir)
                continue
            elif line.type == "INFO":
                fields = line.optional_fields
                # update the current kernel if one is defined in the info
                if "kernel" in fields:
                    current_kernel = line.get_kernel()
                # update the SERVER_JOB reason if one was logged for an abort
                if "job_abort_reason" in fields:
                    running_job.reason = fields["job_abort_reason"]
                    new_tests.append(running_job)
                continue
            elif line.type == "STATUS":
                # update the stacks
                if line.subdir and stack.size() > min_stack_size:
                    subdir_stack[-1] = line.subdir
                # update the status, start and finished times
                stack.update(line.status)
                if status_lib.is_worse_than_or_equal_to(
                        line.status, current_status):
                    if line.reason:
                        # update the status of a currently running test
                        if running_test:
                            running_reasons.add(line.reason)
                            running_reasons = tko_utils.drop_redundant_messages(
                                running_reasons)
                            sorted_reasons = sorted(running_reasons)
                            running_test.reason = ", ".join(sorted_reasons)
                            current_reason = running_test.reason
                            new_tests.append(running_test)
                            logging.debug("update RUNNING reason: %s",
                                          line.reason)
                        else:
                            current_reason = line.reason
                    current_status = stack.current_status()
                started_time = None
                finished_time = line.get_timestamp()
                # if this is a non-test entry there's nothing else to do
                if line.testname is None and line.subdir is None:
                    continue
            elif line.type == "END":
                # grab the current subdir off of the subdir stack, or, if this
                # is the end of a job, just pop it off
                if (line.testname is None and line.subdir is None
                        and not running_test):
                    min_stack_size = stack.size() - 1
                    subdir_stack.pop()
                else:
                    line.subdir = subdir_stack.pop()
                    if not subdir_stack[-1] and stack.size() > min_stack_size:
                        subdir_stack[-1] = line.subdir
                # update the status, start and finished times
                stack.update(line.status)
                current_status = stack.end()
                if stack.size() > min_stack_size:
                    stack.update(current_status)
                    current_status = stack.current_status()
                started_time = started_time_stack.pop()
                finished_time = line.get_timestamp()
                # update the current kernel
                if line.is_successful_reboot(current_status):
                    current_kernel = line.get_kernel()
                # adjust the testname if this is a reboot
                if line.testname == "reboot" and line.subdir is None:
                    line.testname = "boot.%d" % boot_count
            else:
                assert False

            # have we just finished a test?
            if stack.size() <= min_stack_size:
                # if there was no testname, just use the subdir
                if line.testname is None:
                    line.testname = line.subdir
                # if there was no testname or subdir, use 'CLIENT_JOB'
                if line.testname is None:
                    line.testname = "CLIENT_JOB.%d" % job_count
                    running_test = running_client
                    job_count += 1
                    if not status_lib.is_worse_than_or_equal_to(
                            current_status, "ABORT"):
                        # a job hasn't really failed just because some of the
                        # tests it ran have
                        current_status = "GOOD"

                if not current_reason:
                    current_reason = line.reason
                new_test = test.parse_test(self.job, line.subdir,
                                           line.testname, current_status,
                                           current_reason, current_kernel,
                                           started_time, finished_time,
                                           running_test)
                running_test = None
                current_status = status_lib.statuses[-1]
                current_reason = None
                if new_test.testname == ("boot.%d" % boot_count):
                    boot_count += 1

                logging.debug("ADD: %s", new_test.status)
                logging.debug("Subdir: %s", new_test.subdir)
                logging.debug("Testname: %s", new_test.testname)
                logging.debug(new_test.reason)

                new_tests.append(new_test)

        # the job is finished, produce the final SERVER_JOB entry and exit
        final_job = test.parse_test(self.job, "----", "SERVER_JOB",
                                    self.job.exit_status(), running_job.reason,
                                    current_kernel, self.job.started_time,
                                    self.job.finished_time, running_job)
        new_tests.append(final_job)
        yield new_tests
Ejemplo n.º 2
0
 def test_one_unique_message(self):
     self.assertEqual(
         utils.drop_redundant_messages(set(["abc", "abcd", "abcde"])),
         set(["abcde"]))
Ejemplo n.º 3
0
 def test_some_unique_some_not(self):
     self.assertEqual(
         utils.drop_redundant_messages(
             set(["abc", "def", "abcdef", "defghi", "cd"])),
         set(["abcdef", "defghi"]))
Ejemplo n.º 4
0
 def test_singleton(self):
     self.assertEqual(utils.drop_redundant_messages(set(["abc"])),
                      set(["abc"]))
Ejemplo n.º 5
0
 def test_distinct_messages(self):
     self.assertEqual(utils.drop_redundant_messages(set(["abc", "def"])),
                      set(["abc", "def"]))
Ejemplo n.º 6
0
 def test_some_unique_some_not(self):
     self.assertEqual(
         utils.drop_redundant_messages(set(["abc", "def", "abcdef",
                                            "defghi", "cd"])),
         set(["abcdef", "defghi"]))
Ejemplo n.º 7
0
 def test_empty_set(self):
     self.assertEqual(utils.drop_redundant_messages(set()), set())
Ejemplo n.º 8
0
 def test_one_unique_message(self):
     self.assertEqual(
         utils.drop_redundant_messages(set(["abc", "abcd", "abcde"])),
         set(["abcde"]))
Ejemplo n.º 9
0
 def test_distinct_messages(self):
     self.assertEqual(utils.drop_redundant_messages(set(["abc", "def"])),
                      set(["abc", "def"]))
Ejemplo n.º 10
0
 def test_singleton(self):
     self.assertEqual(utils.drop_redundant_messages(set(["abc"])),
                      set(["abc"]))
Ejemplo n.º 11
0
 def test_empty_set(self):
     self.assertEqual(utils.drop_redundant_messages(set()), set())
Ejemplo n.º 12
0
    def state_iterator(self, buffer):
        line = None
        new_tests = []
        job_count, boot_count = 0, 0
        min_stack_size = 0
        stack = status_lib.status_stack()
        current_kernel = kernel("", [])  # UNKNOWN
        current_status = status_lib.statuses[-1]
        current_reason = None
        started_time_stack = [None]
        subdir_stack = [None]
        running_test = None
        running_reasons = set()
        yield []   # we're ready to start running

        # create a RUNNING SERVER_JOB entry to represent the entire test
        running_job = test.parse_partial_test(self.job, "----", "SERVER_JOB",
                                              "", current_kernel,
                                              self.job.started_time)
        new_tests.append(running_job)

        while True:
            # are we finished with parsing?
            if buffer.size() == 0 and self.finished:
                if stack.size() == 0:
                    break
                # we have status lines left on the stack,
                # we need to implicitly abort them first
                logging.debug('Unexpected end of job, aborting')
                abort_subdir_stack = list(subdir_stack)
                if self.job.aborted_by:
                    reason = "Job aborted by %s" % self.job.aborted_by
                    reason += self.job.aborted_on.strftime(
                        " at %b %d %H:%M:%S")
                else:
                    reason = "Job aborted unexpectedly"

                timestamp = line.optional_fields.get('timestamp')
                for i in reversed(xrange(stack.size())):
                    if abort_subdir_stack:
                        subdir = abort_subdir_stack.pop()
                    else:
                        subdir = None
                    abort = self.make_dummy_abort(
                        i, subdir, subdir, timestamp, reason)
                    buffer.put(abort)

            # stop processing once the buffer is empty
            if buffer.size() == 0:
                yield new_tests
                new_tests = []
                continue

            # reinitialize the per-iteration state
            started_time = None
            finished_time = None

            # get the next line
            raw_line = status_lib.clean_raw_line(buffer.get())
            logging.debug('STATUS: %s', raw_line.strip())
            line = status_line.parse_line(raw_line)
            if line is None:
                logging.debug('non-status line, ignoring')
                continue

            # do an initial sanity check of the indentation
            expected_indent = stack.size()
            if line.type == "END":
                expected_indent -= 1
            if line.indent < expected_indent:
                # ABORT the current level if indentation was unexpectedly low
                self.put_back_line_and_abort(
                    buffer, raw_line, stack.size() - 1, subdir_stack[-1],
                    line.optional_fields.get("timestamp"), line.reason)
                continue
            elif line.indent > expected_indent:
                # ignore the log if the indent was unexpectedly high
                logging.debug("unexpected extra indentation, ignoring")
                continue

            # initial line processing
            if line.type == "START":
                stack.start()
                started_time = line.get_timestamp()
                if (line.testname is None and line.subdir is None
                        and not running_test):
                    # we just started a client, all tests are relative to here
                    min_stack_size = stack.size()
                    # start a "RUNNING" CLIENT_JOB entry
                    job_name = "CLIENT_JOB.%d" % job_count
                    running_client = test.parse_partial_test(self.job, None,
                                                             job_name,
                                                             "", current_kernel,
                                                             started_time)
                    logging.debug("RUNNING: %s", running_client.status)
                    logging.debug("Testname: %s", running_client.testname)
                    new_tests.append(running_client)
                elif stack.size() == min_stack_size + 1 and not running_test:
                    # we just started a new test, insert a running record
                    running_reasons = set()
                    if line.reason:
                        running_reasons.add(line.reason)
                    running_test = test.parse_partial_test(self.job,
                                                           line.subdir,
                                                           line.testname,
                                                           line.reason,
                                                           current_kernel,
                                                           started_time)
                    logging.debug("RUNNING: %s", running_test.status)
                    logging.debug("Subdir: %s", running_test.subdir)
                    logging.debug("Testname: %s", running_test.testname)
                    logging.debug("Reason: %s", running_test.reason)
                    new_tests.append(running_test)
                started_time_stack.append(started_time)
                subdir_stack.append(line.subdir)
                continue
            elif line.type == "INFO":
                fields = line.optional_fields
                # update the current kernel if one is defined in the info
                if "kernel" in fields:
                    current_kernel = line.get_kernel()
                # update the SERVER_JOB reason if one was logged for an abort
                if "job_abort_reason" in fields:
                    running_job.reason = fields["job_abort_reason"]
                    new_tests.append(running_job)
                continue
            elif line.type == "STATUS":
                # update the stacks
                if line.subdir and stack.size() > min_stack_size:
                    subdir_stack[-1] = line.subdir
                # update the status, start and finished times
                stack.update(line.status)
                if status_lib.is_worse_than_or_equal_to(line.status,
                                                        current_status):
                    if line.reason:
                        # update the status of a currently running test
                        if running_test:
                            running_reasons.add(line.reason)
                            running_reasons = tko_utils.drop_redundant_messages(
                                running_reasons)
                            sorted_reasons = sorted(running_reasons)
                            running_test.reason = ", ".join(sorted_reasons)
                            current_reason = running_test.reason
                            new_tests.append(running_test)
                            logging.debug("update RUNNING reason: %s",
                                          line.reason)
                        else:
                            current_reason = line.reason
                    current_status = stack.current_status()
                started_time = None
                finished_time = line.get_timestamp()
                # if this is a non-test entry there's nothing else to do
                if line.testname is None and line.subdir is None:
                    continue
            elif line.type == "END":
                # grab the current subdir off of the subdir stack, or, if this
                # is the end of a job, just pop it off
                if (line.testname is None and line.subdir is None
                        and not running_test):
                    min_stack_size = stack.size() - 1
                    subdir_stack.pop()
                else:
                    line.subdir = subdir_stack.pop()
                    if not subdir_stack[-1] and stack.size() > min_stack_size:
                        subdir_stack[-1] = line.subdir
                # update the status, start and finished times
                stack.update(line.status)
                current_status = stack.end()
                if stack.size() > min_stack_size:
                    stack.update(current_status)
                    current_status = stack.current_status()
                started_time = started_time_stack.pop()
                finished_time = line.get_timestamp()
                # update the current kernel
                if line.is_successful_reboot(current_status):
                    current_kernel = line.get_kernel()
                # adjust the testname if this is a reboot
                if line.testname == "reboot" and line.subdir is None:
                    line.testname = "boot.%d" % boot_count
            else:
                assert False

            # have we just finished a test?
            if stack.size() <= min_stack_size:
                # if there was no testname, just use the subdir
                if line.testname is None:
                    line.testname = line.subdir
                # if there was no testname or subdir, use 'CLIENT_JOB'
                if line.testname is None:
                    line.testname = "CLIENT_JOB.%d" % job_count
                    running_test = running_client
                    job_count += 1
                    if not status_lib.is_worse_than_or_equal_to(
                            current_status, "ABORT"):
                        # a job hasn't really failed just because some of the
                        # tests it ran have
                        current_status = "GOOD"

                if not current_reason:
                    current_reason = line.reason
                new_test = test.parse_test(self.job,
                                           line.subdir,
                                           line.testname,
                                           current_status,
                                           current_reason,
                                           current_kernel,
                                           started_time,
                                           finished_time,
                                           running_test)
                running_test = None
                current_status = status_lib.statuses[-1]
                current_reason = None
                if new_test.testname == ("boot.%d" % boot_count):
                    boot_count += 1

                logging.debug("ADD: %s", new_test.status)
                logging.debug("Subdir: %s", new_test.subdir)
                logging.debug("Testname: %s", new_test.testname)
                logging.debug(new_test.reason)

                new_tests.append(new_test)

        # the job is finished, produce the final SERVER_JOB entry and exit
        final_job = test.parse_test(self.job, "----", "SERVER_JOB",
                                    self.job.exit_status(), running_job.reason,
                                    current_kernel,
                                    self.job.started_time,
                                    self.job.finished_time,
                                    running_job)
        new_tests.append(final_job)
        yield new_tests