def test_find_python_traceback(self): def run(*args): return Popen(args, stdout=PIPE, stderr=PIPE).communicate() # sanity-check normal operations ok_stdout, ok_stderr = run('python', '-c', "print sorted('321')") self.assertEqual(ok_stdout.rstrip(), "['1', '2', '3']") self.assertEqual(find_python_traceback(StringIO(ok_stderr)), None) # Oops, can't sort a number. stdout, stderr = run('python', '-c', "print sorted(321)") # We expect something like this: # # Traceback (most recent call last): # File "<string>", line 1, in <module> # TypeError: 'int' object is not iterable self.assertEqual(stdout, '') # save the traceback for the next step tb = find_python_traceback(StringIO(stderr)) self.assertNotEqual(tb, None) assert isinstance(tb, list) # The first line ("Traceback...") is not skipped self.assertEqual(len(tb), 3) # make sure we can find the same traceback in noise verbose_stdout, verbose_stderr = run( 'python', '-v', '-c', "print sorted(321)") self.assertEqual(verbose_stdout, '') self.assertNotEqual(verbose_stderr, stderr) verbose_tb = find_python_traceback(StringIO(verbose_stderr)) self.assertEqual(verbose_tb, tb)
def _wait_for_process(self, proc_dict, step_num): # handle counters, status msgs, and other stuff on stderr proc = proc_dict['proc'] stderr_lines = self._process_stderr_from_script( proc.stderr, step_num=step_num) tb_lines = find_python_traceback(stderr_lines) # proc.stdout isn't always defined if proc.stdout: proc.stdout.close() proc.stderr.close() returncode = proc.wait() if returncode != 0: self.print_counters([step_num + 1]) # try to throw a useful exception if tb_lines: raise Exception( 'Command %r returned non-zero exit status %d:\n%s' % (proc_dict['args'], returncode, ''.join(tb_lines))) else: raise Exception( 'Command %r returned non-zero exit status %d' % (proc_dict['args'], returncode))
def _wait_for_process(self, proc_dict, step_num): # handle counters, status msgs, and other stuff on stderr proc = proc_dict['proc'] stderr_lines = self._process_stderr_from_script(proc.stderr, step_num=step_num) tb_lines = find_python_traceback(stderr_lines) # proc.stdout isn't always defined if proc.stdout: proc.stdout.close() proc.stderr.close() returncode = proc.wait() if returncode != 0: # show counters before raising exception counters = self._counters[step_num] if counters: log.info(_format_counters(counters)) # try to throw a useful exception if tb_lines: for line in tb_lines: log.error(line.rstrip('\r\n')) reason = str(CalledProcessError(returncode, proc_dict['args'])) raise StepFailedException(reason=reason, step_num=step_num, num_steps=len(self._get_steps()))
def _wait_for_process(self, proc_dict, step_num): # handle counters, status msgs, and other stuff on stderr proc = proc_dict['proc'] stderr_lines = self._process_stderr_from_script( proc.stderr, step_num=step_num) tb_lines = find_python_traceback(stderr_lines) # proc.stdout isn't always defined if proc.stdout: proc.stdout.close() proc.stderr.close() returncode = proc.wait() if returncode != 0: # show counters before raising exception counters = self._counters[step_num] if counters: log.info(_format_counters(counters)) # try to throw a useful exception if tb_lines: for line in tb_lines: log.error(line.rstrip('\r\n')) reason = str( CalledProcessError(returncode, proc_dict['args'])) raise StepFailedException( reason=reason, step_num=step_num, num_steps=len(self._get_steps()))
def _wait_for_process(self, proc_dict, step_num): # handle counters, status msgs, and other stuff on stderr proc = proc_dict['proc'] stderr_lines = self._process_stderr_from_script(proc.stderr, step_num=step_num) tb_lines = find_python_traceback(stderr_lines) # proc.stdout isn't always defined if proc.stdout: proc.stdout.close() proc.stderr.close() returncode = proc.wait() if returncode != 0: self._print_counters(step_nums=[step_num]) # try to throw a useful exception if tb_lines: raise Exception( 'Command %r returned non-zero exit status %d:\n%s' % (proc_dict['args'], returncode, ''.join(tb_lines))) else: raise Exception('Command %r returned non-zero exit status %d' % (proc_dict['args'], returncode))
def test_find_python_traceback_with_more_stderr_2(self): total_traceback = self.EXAMPLE_STDERR_TRACEBACK_2 + 'junk\n' tb = find_python_traceback(StringIO(total_traceback)) self.assertEqual(''.join(tb), self.EXAMPLE_STDERR_TRACEBACK_2)
def test_find_multiple_python_tracebacks(self): total_traceback = self.EXAMPLE_TRACEBACK + 'junk\n' tb = find_python_traceback(StringIO(total_traceback)) self.assertEqual(''.join(tb), self.EXAMPLE_TRACEBACK)
def test_find_python_traceback_with_more_stderr_2(self): total_traceback = self.EXAMPLE_STDERR_TRACEBACK_2 + b'junk\n' tb = find_python_traceback(BytesIO(total_traceback)) self.assertEqual(''.join(tb), self.EXAMPLE_STDERR_TRACEBACK_2.decode('ascii'))
def test_find_multiple_python_tracebacks(self): total_traceback = self.EXAMPLE_TRACEBACK + b'junk\n' tb = find_python_traceback(BytesIO(total_traceback)) self.assertEqual(''.join(tb), self.EXAMPLE_TRACEBACK.decode('ascii'))
def parse(self, lines): return find_python_traceback(lines)
def _invoke_step(self, args, outfile_name, env=None): """Run the given command, outputting into outfile, and reading from the previous outfile (or, for the first step, from our original output files). outfile is a path relative to our local tmp dir. commands are run inside self._working_dir We'll intelligently handle stderr from the process. """ # keep the current environment because we need PATH to find binaries # and make PYTHONPATH work env = combine_local_envs( {'PYTHONPATH': os.getcwd()}, os.environ, self._get_cmdenv(), env or {}) # decide where to get input if self._prev_outfile is not None: input_paths = [self._prev_outfile] else: input_paths = [] for path in self._input_paths: if path == '-': input_paths.append(self._dump_stdin_to_local_file()) else: input_paths.append(path) # add input to the command line for path in input_paths: args.append(os.path.abspath(path)) log.info('> %s' % cmd_line(args)) # set up outfile outfile = os.path.join(self._get_local_tmp_dir(), outfile_name) log.info('writing to %s' % outfile) log.debug('') self._prev_outfile = outfile write_to = open(outfile, 'w') # run the process proc = Popen(args, stdout=write_to, stderr=PIPE, cwd=self._working_dir, env=env) # handle counters, status msgs, and other stuff on stderr stderr_lines = self._process_stderr_from_script(proc.stderr) tb_lines = find_python_traceback(stderr_lines) self._print_counters() returncode = proc.wait() if returncode != 0: # try to throw a useful exception if tb_lines: raise Exception( 'Command %r returned non-zero exit status %d:\n%s' % (args, returncode, ''.join(tb_lines))) else: raise Exception( 'Command %r returned non-zero exit status %d: %s' % (args, returncode)) # flush file descriptors write_to.flush()
def _invoke_step(self, args, outfile_name, env=None): """Run the given command, outputting into outfile, and reading from the previous outfile (or, for the first step, from our original output files). outfile is a path relative to our local tmp dir. commands are run inside self._working_dir We'll intelligently handle stderr from the process. """ # keep the current environment because we need PATH to find binaries # and make PYTHONPATH work env = combine_local_envs({'PYTHONPATH': os.getcwd()}, os.environ, self._get_cmdenv(), env or {}) # decide where to get input if self._prev_outfile is not None: input_paths = [self._prev_outfile] else: input_paths = [] for path in self._input_paths: if path == '-': input_paths.append(self._dump_stdin_to_local_file()) else: input_paths.append(path) # add input to the command line for path in input_paths: args.append(os.path.abspath(path)) log.info('> %s' % cmd_line(args)) # set up outfile outfile = os.path.join(self._get_local_tmp_dir(), outfile_name) log.info('writing to %s' % outfile) log.debug('') self._prev_outfile = outfile write_to = open(outfile, 'w') # run the process proc = Popen(args, stdout=write_to, stderr=PIPE, cwd=self._working_dir, env=env) # handle counters, status msgs, and other stuff on stderr stderr_lines = self._process_stderr_from_script(proc.stderr) tb_lines = find_python_traceback(stderr_lines) self._print_counters() returncode = proc.wait() if returncode != 0: # try to throw a useful exception if tb_lines: raise Exception( 'Command %r returned non-zero exit status %d:\n%s' % (args, returncode, ''.join(tb_lines))) else: raise Exception( 'Command %r returned non-zero exit status %d: %s' % (args, returncode)) # flush file descriptors write_to.flush()