def test_find_python_traceback(self):
def run(*args):
return Popen(args, stdout=PIPE, stderr=PIPE).communicate()
# sanity-check normal operations
ok_stdout, ok_stderr = run('python', '-c', "print sorted('321')")
self.assertEqual(ok_stdout.rstrip(), "['1', '2', '3']")
self.assertEqual(find_python_traceback(StringIO(ok_stderr)), None)
# Oops, can't sort a number.
stdout, stderr = run('python', '-c', "print sorted(321)")
# We expect something like this:
#
# Traceback (most recent call last):
# File "<string>", line 1, in <module>
# TypeError: 'int' object is not iterable
self.assertEqual(stdout, '')
# save the traceback for the next step
tb = find_python_traceback(StringIO(stderr))
self.assertNotEqual(tb, None)
assert isinstance(tb, list)
# The first line ("Traceback...") is not skipped
self.assertEqual(len(tb), 3)
# make sure we can find the same traceback in noise
verbose_stdout, verbose_stderr = run(
'python', '-v', '-c', "print sorted(321)")
self.assertEqual(verbose_stdout, '')
self.assertNotEqual(verbose_stderr, stderr)
verbose_tb = find_python_traceback(StringIO(verbose_stderr))
self.assertEqual(verbose_tb, tb)
def test_find_python_traceback(self):
def run(*args):
return Popen(args, stdout=PIPE, stderr=PIPE).communicate()
# sanity-check normal operations
ok_stdout, ok_stderr = run('python', '-c', "print sorted('321')")
self.assertEqual(ok_stdout.rstrip(), "['1', '2', '3']")
self.assertEqual(find_python_traceback(StringIO(ok_stderr)), None)
# Oops, can't sort a number.
stdout, stderr = run('python', '-c', "print sorted(321)")
# We expect something like this:
#
# Traceback (most recent call last):
# File "<string>", line 1, in <module>
# TypeError: 'int' object is not iterable
self.assertEqual(stdout, '')
# save the traceback for the next step
tb = find_python_traceback(StringIO(stderr))
self.assertNotEqual(tb, None)
assert isinstance(tb, list)
# The first line ("Traceback...") is not skipped
self.assertEqual(len(tb), 3)
# make sure we can find the same traceback in noise
verbose_stdout, verbose_stderr = run(
'python', '-v', '-c', "print sorted(321)")
self.assertEqual(verbose_stdout, '')
self.assertNotEqual(verbose_stderr, stderr)
verbose_tb = find_python_traceback(StringIO(verbose_stderr))
self.assertEqual(verbose_tb, tb)
def _wait_for_process(self, proc_dict, step_num):
# handle counters, status msgs, and other stuff on stderr
proc = proc_dict['proc']
stderr_lines = self._process_stderr_from_script(
proc.stderr, step_num=step_num)
tb_lines = find_python_traceback(stderr_lines)
# proc.stdout isn't always defined
if proc.stdout:
proc.stdout.close()
proc.stderr.close()
returncode = proc.wait()
if returncode != 0:
self.print_counters([step_num + 1])
# try to throw a useful exception
if tb_lines:
raise Exception(
'Command %r returned non-zero exit status %d:\n%s' %
(proc_dict['args'], returncode, ''.join(tb_lines)))
else:
raise Exception(
'Command %r returned non-zero exit status %d' %
(proc_dict['args'], returncode))
def _wait_for_process(self, proc_dict, step_num):
# handle counters, status msgs, and other stuff on stderr
proc = proc_dict['proc']
stderr_lines = self._process_stderr_from_script(proc.stderr,
step_num=step_num)
tb_lines = find_python_traceback(stderr_lines)
# proc.stdout isn't always defined
if proc.stdout:
proc.stdout.close()
proc.stderr.close()
returncode = proc.wait()
if returncode != 0:
# show counters before raising exception
counters = self._counters[step_num]
if counters:
log.info(_format_counters(counters))
# try to throw a useful exception
if tb_lines:
for line in tb_lines:
log.error(line.rstrip('\r\n'))
reason = str(CalledProcessError(returncode, proc_dict['args']))
raise StepFailedException(reason=reason,
step_num=step_num,
num_steps=len(self._get_steps()))
def _wait_for_process(self, proc_dict, step_num):
# handle counters, status msgs, and other stuff on stderr
proc = proc_dict['proc']
stderr_lines = self._process_stderr_from_script(
proc.stderr, step_num=step_num)
tb_lines = find_python_traceback(stderr_lines)
# proc.stdout isn't always defined
if proc.stdout:
proc.stdout.close()
proc.stderr.close()
returncode = proc.wait()
if returncode != 0:
# show counters before raising exception
counters = self._counters[step_num]
if counters:
log.info(_format_counters(counters))
# try to throw a useful exception
if tb_lines:
for line in tb_lines:
log.error(line.rstrip('\r\n'))
reason = str(
CalledProcessError(returncode, proc_dict['args']))
raise StepFailedException(
reason=reason, step_num=step_num,
num_steps=len(self._get_steps()))
def _wait_for_process(self, proc_dict, step_num):
# handle counters, status msgs, and other stuff on stderr
proc = proc_dict['proc']
stderr_lines = self._process_stderr_from_script(proc.stderr,
step_num=step_num)
tb_lines = find_python_traceback(stderr_lines)
# proc.stdout isn't always defined
if proc.stdout:
proc.stdout.close()
proc.stderr.close()
returncode = proc.wait()
if returncode != 0:
self._print_counters(step_nums=[step_num])
# try to throw a useful exception
if tb_lines:
raise Exception(
'Command %r returned non-zero exit status %d:\n%s' %
(proc_dict['args'], returncode, ''.join(tb_lines)))
else:
raise Exception('Command %r returned non-zero exit status %d' %
(proc_dict['args'], returncode))
def test_find_python_traceback_with_more_stderr_2(self):
total_traceback = self.EXAMPLE_STDERR_TRACEBACK_2 + 'junk\n'
tb = find_python_traceback(StringIO(total_traceback))
self.assertEqual(''.join(tb), self.EXAMPLE_STDERR_TRACEBACK_2)
def test_find_multiple_python_tracebacks(self):
total_traceback = self.EXAMPLE_TRACEBACK + 'junk\n'
tb = find_python_traceback(StringIO(total_traceback))
self.assertEqual(''.join(tb), self.EXAMPLE_TRACEBACK)
def test_find_python_traceback_with_more_stderr_2(self):
total_traceback = self.EXAMPLE_STDERR_TRACEBACK_2 + 'junk\n'
tb = find_python_traceback(StringIO(total_traceback))
self.assertEqual(''.join(tb), self.EXAMPLE_STDERR_TRACEBACK_2)
def test_find_multiple_python_tracebacks(self):
total_traceback = self.EXAMPLE_TRACEBACK + 'junk\n'
tb = find_python_traceback(StringIO(total_traceback))
self.assertEqual(''.join(tb), self.EXAMPLE_TRACEBACK)
def test_find_python_traceback_with_more_stderr_2(self):
total_traceback = self.EXAMPLE_STDERR_TRACEBACK_2 + b'junk\n'
tb = find_python_traceback(BytesIO(total_traceback))
self.assertEqual(''.join(tb),
self.EXAMPLE_STDERR_TRACEBACK_2.decode('ascii'))
def test_find_multiple_python_tracebacks(self):
total_traceback = self.EXAMPLE_TRACEBACK + b'junk\n'
tb = find_python_traceback(BytesIO(total_traceback))
self.assertEqual(''.join(tb), self.EXAMPLE_TRACEBACK.decode('ascii'))
def test_find_python_traceback_with_more_stderr_2(self):
total_traceback = self.EXAMPLE_STDERR_TRACEBACK_2 + b'junk\n'
tb = find_python_traceback(BytesIO(total_traceback))
self.assertEqual(''.join(tb),
self.EXAMPLE_STDERR_TRACEBACK_2.decode('ascii'))
def test_find_multiple_python_tracebacks(self):
total_traceback = self.EXAMPLE_TRACEBACK + b'junk\n'
tb = find_python_traceback(BytesIO(total_traceback))
self.assertEqual(''.join(tb),
self.EXAMPLE_TRACEBACK.decode('ascii'))
def parse(self, lines):
return find_python_traceback(lines)
def parse(self, lines):
return find_python_traceback(lines)
def _invoke_step(self, args, outfile_name, env=None):
"""Run the given command, outputting into outfile, and reading
from the previous outfile (or, for the first step, from our
original output files).
outfile is a path relative to our local tmp dir. commands are run
inside self._working_dir
We'll intelligently handle stderr from the process.
"""
# keep the current environment because we need PATH to find binaries
# and make PYTHONPATH work
env = combine_local_envs(
{'PYTHONPATH': os.getcwd()},
os.environ,
self._get_cmdenv(),
env or {})
# decide where to get input
if self._prev_outfile is not None:
input_paths = [self._prev_outfile]
else:
input_paths = []
for path in self._input_paths:
if path == '-':
input_paths.append(self._dump_stdin_to_local_file())
else:
input_paths.append(path)
# add input to the command line
for path in input_paths:
args.append(os.path.abspath(path))
log.info('> %s' % cmd_line(args))
# set up outfile
outfile = os.path.join(self._get_local_tmp_dir(), outfile_name)
log.info('writing to %s' % outfile)
log.debug('')
self._prev_outfile = outfile
write_to = open(outfile, 'w')
# run the process
proc = Popen(args, stdout=write_to, stderr=PIPE,
cwd=self._working_dir, env=env)
# handle counters, status msgs, and other stuff on stderr
stderr_lines = self._process_stderr_from_script(proc.stderr)
tb_lines = find_python_traceback(stderr_lines)
self._print_counters()
returncode = proc.wait()
if returncode != 0:
# try to throw a useful exception
if tb_lines:
raise Exception(
'Command %r returned non-zero exit status %d:\n%s' %
(args, returncode, ''.join(tb_lines)))
else:
raise Exception(
'Command %r returned non-zero exit status %d: %s' %
(args, returncode))
# flush file descriptors
write_to.flush()
def _invoke_step(self, args, outfile_name, env=None):
"""Run the given command, outputting into outfile, and reading
from the previous outfile (or, for the first step, from our
original output files).
outfile is a path relative to our local tmp dir. commands are run
inside self._working_dir
We'll intelligently handle stderr from the process.
"""
# keep the current environment because we need PATH to find binaries
# and make PYTHONPATH work
env = combine_local_envs({'PYTHONPATH': os.getcwd()}, os.environ,
self._get_cmdenv(), env or {})
# decide where to get input
if self._prev_outfile is not None:
input_paths = [self._prev_outfile]
else:
input_paths = []
for path in self._input_paths:
if path == '-':
input_paths.append(self._dump_stdin_to_local_file())
else:
input_paths.append(path)
# add input to the command line
for path in input_paths:
args.append(os.path.abspath(path))
log.info('> %s' % cmd_line(args))
# set up outfile
outfile = os.path.join(self._get_local_tmp_dir(), outfile_name)
log.info('writing to %s' % outfile)
log.debug('')
self._prev_outfile = outfile
write_to = open(outfile, 'w')
# run the process
proc = Popen(args,
stdout=write_to,
stderr=PIPE,
cwd=self._working_dir,
env=env)
# handle counters, status msgs, and other stuff on stderr
stderr_lines = self._process_stderr_from_script(proc.stderr)
tb_lines = find_python_traceback(stderr_lines)
self._print_counters()
returncode = proc.wait()
if returncode != 0:
# try to throw a useful exception
if tb_lines:
raise Exception(
'Command %r returned non-zero exit status %d:\n%s' %
(args, returncode, ''.join(tb_lines)))
else:
raise Exception(
'Command %r returned non-zero exit status %d: %s' %
(args, returncode))
# flush file descriptors
write_to.flush()
