def state_iterator(self, buffer):
line = None
new_tests = []
job_count, boot_count = 0, 0
min_stack_size = 0
stack = status_lib.status_stack()
current_kernel = kernel("", []) # UNKNOWN
current_status = status_lib.statuses[-1]
current_reason = None
started_time_stack = [None]
subdir_stack = [None]
running_test = None
running_reasons = set()
yield [] # we're ready to start running
# create a RUNNING SERVER_JOB entry to represent the entire test
running_job = test.parse_partial_test(self.job, "----", "SERVER_JOB",
"", current_kernel,
self.job.started_time)
new_tests.append(running_job)
while True:
# are we finished with parsing?
if buffer.size() == 0 and self.finished:
if stack.size() == 0:
break
# we have status lines left on the stack,
# we need to implicitly abort them first
logging.debug('Unexpected end of job, aborting')
abort_subdir_stack = list(subdir_stack)
if self.job.aborted_by:
reason = "Job aborted by %s" % self.job.aborted_by
reason += self.job.aborted_on.strftime(
" at %b %d %H:%M:%S")
else:
reason = "Job aborted unexpectedly"
timestamp = line.optional_fields.get('timestamp')
for i in reversed(xrange(stack.size())):
if abort_subdir_stack:
subdir = abort_subdir_stack.pop()
else:
subdir = None
abort = self.make_dummy_abort(i, subdir, subdir, timestamp,
reason)
buffer.put(abort)
# stop processing once the buffer is empty
if buffer.size() == 0:
yield new_tests
new_tests = []
continue
# reinitialize the per-iteration state
started_time = None
finished_time = None
# get the next line
raw_line = status_lib.clean_raw_line(buffer.get())
logging.debug('STATUS: %s', raw_line.strip())
line = status_line.parse_line(raw_line)
if line is None:
logging.debug('non-status line, ignoring')
continue
# do an initial sanity check of the indentation
expected_indent = stack.size()
if line.type == "END":
expected_indent -= 1
if line.indent < expected_indent:
# ABORT the current level if indentation was unexpectedly low
self.put_back_line_and_abort(
buffer, raw_line,
stack.size() - 1, subdir_stack[-1],
line.optional_fields.get("timestamp"), line.reason)
continue
elif line.indent > expected_indent:
# ignore the log if the indent was unexpectedly high
logging.debug("unexpected extra indentation, ignoring")
continue
# initial line processing
if line.type == "START":
stack.start()
started_time = line.get_timestamp()
if (line.testname is None and line.subdir is None
and not running_test):
# we just started a client, all tests are relative to here
min_stack_size = stack.size()
# start a "RUNNING" CLIENT_JOB entry
job_name = "CLIENT_JOB.%d" % job_count
running_client = test.parse_partial_test(
self.job, None, job_name, "", current_kernel,
started_time)
logging.debug("RUNNING: %s", running_client.status)
logging.debug("Testname: %s", running_client.testname)
new_tests.append(running_client)
elif stack.size() == min_stack_size + 1 and not running_test:
# we just started a new test, insert a running record
running_reasons = set()
if line.reason:
running_reasons.add(line.reason)
running_test = test.parse_partial_test(
self.job, line.subdir, line.testname, line.reason,
current_kernel, started_time)
logging.debug("RUNNING: %s", running_test.status)
logging.debug("Subdir: %s", running_test.subdir)
logging.debug("Testname: %s", running_test.testname)
logging.debug("Reason: %s", running_test.reason)
new_tests.append(running_test)
started_time_stack.append(started_time)
subdir_stack.append(line.subdir)
continue
elif line.type == "INFO":
fields = line.optional_fields
# update the current kernel if one is defined in the info
if "kernel" in fields:
current_kernel = line.get_kernel()
# update the SERVER_JOB reason if one was logged for an abort
if "job_abort_reason" in fields:
running_job.reason = fields["job_abort_reason"]
new_tests.append(running_job)
continue
elif line.type == "STATUS":
# update the stacks
if line.subdir and stack.size() > min_stack_size:
subdir_stack[-1] = line.subdir
# update the status, start and finished times
stack.update(line.status)
if status_lib.is_worse_than_or_equal_to(
line.status, current_status):
if line.reason:
# update the status of a currently running test
if running_test:
running_reasons.add(line.reason)
running_reasons = tko_utils.drop_redundant_messages(
running_reasons)
sorted_reasons = sorted(running_reasons)
running_test.reason = ", ".join(sorted_reasons)
current_reason = running_test.reason
new_tests.append(running_test)
logging.debug("update RUNNING reason: %s",
line.reason)
else:
current_reason = line.reason
current_status = stack.current_status()
started_time = None
finished_time = line.get_timestamp()
# if this is a non-test entry there's nothing else to do
if line.testname is None and line.subdir is None:
continue
elif line.type == "END":
# grab the current subdir off of the subdir stack, or, if this
# is the end of a job, just pop it off
if (line.testname is None and line.subdir is None
and not running_test):
min_stack_size = stack.size() - 1
subdir_stack.pop()
else:
line.subdir = subdir_stack.pop()
if not subdir_stack[-1] and stack.size() > min_stack_size:
subdir_stack[-1] = line.subdir
# update the status, start and finished times
stack.update(line.status)
current_status = stack.end()
if stack.size() > min_stack_size:
stack.update(current_status)
current_status = stack.current_status()
started_time = started_time_stack.pop()
finished_time = line.get_timestamp()
# update the current kernel
if line.is_successful_reboot(current_status):
current_kernel = line.get_kernel()
# adjust the testname if this is a reboot
if line.testname == "reboot" and line.subdir is None:
line.testname = "boot.%d" % boot_count
else:
assert False
# have we just finished a test?
if stack.size() <= min_stack_size:
# if there was no testname, just use the subdir
if line.testname is None:
line.testname = line.subdir
# if there was no testname or subdir, use 'CLIENT_JOB'
if line.testname is None:
line.testname = "CLIENT_JOB.%d" % job_count
running_test = running_client
job_count += 1
if not status_lib.is_worse_than_or_equal_to(
current_status, "ABORT"):
# a job hasn't really failed just because some of the
# tests it ran have
current_status = "GOOD"
if not current_reason:
current_reason = line.reason
new_test = test.parse_test(self.job, line.subdir,
line.testname, current_status,
current_reason, current_kernel,
started_time, finished_time,
running_test)
running_test = None
current_status = status_lib.statuses[-1]
current_reason = None
if new_test.testname == ("boot.%d" % boot_count):
boot_count += 1
logging.debug("ADD: %s", new_test.status)
logging.debug("Subdir: %s", new_test.subdir)
logging.debug("Testname: %s", new_test.testname)
logging.debug(new_test.reason)
new_tests.append(new_test)
# the job is finished, produce the final SERVER_JOB entry and exit
final_job = test.parse_test(self.job, "----", "SERVER_JOB",
self.job.exit_status(), running_job.reason,
current_kernel, self.job.started_time,
self.job.finished_time, running_job)
new_tests.append(final_job)
yield new_tests
def test_one_unique_message(self):
self.assertEqual(
utils.drop_redundant_messages(set(["abc", "abcd", "abcde"])),
set(["abcde"]))
def test_some_unique_some_not(self):
self.assertEqual(
utils.drop_redundant_messages(
set(["abc", "def", "abcdef", "defghi", "cd"])),
set(["abcdef", "defghi"]))
def test_singleton(self):
self.assertEqual(utils.drop_redundant_messages(set(["abc"])),
set(["abc"]))
def test_distinct_messages(self):
self.assertEqual(utils.drop_redundant_messages(set(["abc", "def"])),
set(["abc", "def"]))
def test_some_unique_some_not(self):
self.assertEqual(
utils.drop_redundant_messages(set(["abc", "def", "abcdef",
"defghi", "cd"])),
set(["abcdef", "defghi"]))
def test_empty_set(self):
self.assertEqual(utils.drop_redundant_messages(set()), set())
def test_one_unique_message(self):
self.assertEqual(
utils.drop_redundant_messages(set(["abc", "abcd", "abcde"])),
set(["abcde"]))
def test_distinct_messages(self):
self.assertEqual(utils.drop_redundant_messages(set(["abc", "def"])),
set(["abc", "def"]))
def test_singleton(self):
self.assertEqual(utils.drop_redundant_messages(set(["abc"])),
set(["abc"]))
def test_empty_set(self):
self.assertEqual(utils.drop_redundant_messages(set()), set())
def state_iterator(self, buffer):
line = None
new_tests = []
job_count, boot_count = 0, 0
min_stack_size = 0
stack = status_lib.status_stack()
current_kernel = kernel("", []) # UNKNOWN
current_status = status_lib.statuses[-1]
current_reason = None
started_time_stack = [None]
subdir_stack = [None]
running_test = None
running_reasons = set()
yield [] # we're ready to start running
# create a RUNNING SERVER_JOB entry to represent the entire test
running_job = test.parse_partial_test(self.job, "----", "SERVER_JOB",
"", current_kernel,
self.job.started_time)
new_tests.append(running_job)
while True:
# are we finished with parsing?
if buffer.size() == 0 and self.finished:
if stack.size() == 0:
break
# we have status lines left on the stack,
# we need to implicitly abort them first
logging.debug('Unexpected end of job, aborting')
abort_subdir_stack = list(subdir_stack)
if self.job.aborted_by:
reason = "Job aborted by %s" % self.job.aborted_by
reason += self.job.aborted_on.strftime(
" at %b %d %H:%M:%S")
else:
reason = "Job aborted unexpectedly"
timestamp = line.optional_fields.get('timestamp')
for i in reversed(xrange(stack.size())):
if abort_subdir_stack:
subdir = abort_subdir_stack.pop()
else:
subdir = None
abort = self.make_dummy_abort(
i, subdir, subdir, timestamp, reason)
buffer.put(abort)
# stop processing once the buffer is empty
if buffer.size() == 0:
yield new_tests
new_tests = []
continue
# reinitialize the per-iteration state
started_time = None
finished_time = None
# get the next line
raw_line = status_lib.clean_raw_line(buffer.get())
logging.debug('STATUS: %s', raw_line.strip())
line = status_line.parse_line(raw_line)
if line is None:
logging.debug('non-status line, ignoring')
continue
# do an initial sanity check of the indentation
expected_indent = stack.size()
if line.type == "END":
expected_indent -= 1
if line.indent < expected_indent:
# ABORT the current level if indentation was unexpectedly low
self.put_back_line_and_abort(
buffer, raw_line, stack.size() - 1, subdir_stack[-1],
line.optional_fields.get("timestamp"), line.reason)
continue
elif line.indent > expected_indent:
# ignore the log if the indent was unexpectedly high
logging.debug("unexpected extra indentation, ignoring")
continue
# initial line processing
if line.type == "START":
stack.start()
started_time = line.get_timestamp()
if (line.testname is None and line.subdir is None
and not running_test):
# we just started a client, all tests are relative to here
min_stack_size = stack.size()
# start a "RUNNING" CLIENT_JOB entry
job_name = "CLIENT_JOB.%d" % job_count
running_client = test.parse_partial_test(self.job, None,
job_name,
"", current_kernel,
started_time)
logging.debug("RUNNING: %s", running_client.status)
logging.debug("Testname: %s", running_client.testname)
new_tests.append(running_client)
elif stack.size() == min_stack_size + 1 and not running_test:
# we just started a new test, insert a running record
running_reasons = set()
if line.reason:
running_reasons.add(line.reason)
running_test = test.parse_partial_test(self.job,
line.subdir,
line.testname,
line.reason,
current_kernel,
started_time)
logging.debug("RUNNING: %s", running_test.status)
logging.debug("Subdir: %s", running_test.subdir)
logging.debug("Testname: %s", running_test.testname)
logging.debug("Reason: %s", running_test.reason)
new_tests.append(running_test)
started_time_stack.append(started_time)
subdir_stack.append(line.subdir)
continue
elif line.type == "INFO":
fields = line.optional_fields
# update the current kernel if one is defined in the info
if "kernel" in fields:
current_kernel = line.get_kernel()
# update the SERVER_JOB reason if one was logged for an abort
if "job_abort_reason" in fields:
running_job.reason = fields["job_abort_reason"]
new_tests.append(running_job)
continue
elif line.type == "STATUS":
# update the stacks
if line.subdir and stack.size() > min_stack_size:
subdir_stack[-1] = line.subdir
# update the status, start and finished times
stack.update(line.status)
if status_lib.is_worse_than_or_equal_to(line.status,
current_status):
if line.reason:
# update the status of a currently running test
if running_test:
running_reasons.add(line.reason)
running_reasons = tko_utils.drop_redundant_messages(
running_reasons)
sorted_reasons = sorted(running_reasons)
running_test.reason = ", ".join(sorted_reasons)
current_reason = running_test.reason
new_tests.append(running_test)
logging.debug("update RUNNING reason: %s",
line.reason)
else:
current_reason = line.reason
current_status = stack.current_status()
started_time = None
finished_time = line.get_timestamp()
# if this is a non-test entry there's nothing else to do
if line.testname is None and line.subdir is None:
continue
elif line.type == "END":
# grab the current subdir off of the subdir stack, or, if this
# is the end of a job, just pop it off
if (line.testname is None and line.subdir is None
and not running_test):
min_stack_size = stack.size() - 1
subdir_stack.pop()
else:
line.subdir = subdir_stack.pop()
if not subdir_stack[-1] and stack.size() > min_stack_size:
subdir_stack[-1] = line.subdir
# update the status, start and finished times
stack.update(line.status)
current_status = stack.end()
if stack.size() > min_stack_size:
stack.update(current_status)
current_status = stack.current_status()
started_time = started_time_stack.pop()
finished_time = line.get_timestamp()
# update the current kernel
if line.is_successful_reboot(current_status):
current_kernel = line.get_kernel()
# adjust the testname if this is a reboot
if line.testname == "reboot" and line.subdir is None:
line.testname = "boot.%d" % boot_count
else:
assert False
# have we just finished a test?
if stack.size() <= min_stack_size:
# if there was no testname, just use the subdir
if line.testname is None:
line.testname = line.subdir
# if there was no testname or subdir, use 'CLIENT_JOB'
if line.testname is None:
line.testname = "CLIENT_JOB.%d" % job_count
running_test = running_client
job_count += 1
if not status_lib.is_worse_than_or_equal_to(
current_status, "ABORT"):
# a job hasn't really failed just because some of the
# tests it ran have
current_status = "GOOD"
if not current_reason:
current_reason = line.reason
new_test = test.parse_test(self.job,
line.subdir,
line.testname,
current_status,
current_reason,
current_kernel,
started_time,
finished_time,
running_test)
running_test = None
current_status = status_lib.statuses[-1]
current_reason = None
if new_test.testname == ("boot.%d" % boot_count):
boot_count += 1
logging.debug("ADD: %s", new_test.status)
logging.debug("Subdir: %s", new_test.subdir)
logging.debug("Testname: %s", new_test.testname)
logging.debug(new_test.reason)
new_tests.append(new_test)
# the job is finished, produce the final SERVER_JOB entry and exit
final_job = test.parse_test(self.job, "----", "SERVER_JOB",
self.job.exit_status(), running_job.reason,
current_kernel,
self.job.started_time,
self.job.finished_time,
running_job)
new_tests.append(final_job)
yield new_tests