def run_nosetests(nose_cmd, first, last): """ Run nosetests and return the output :param nose_cmd: The nosetests command, with all parameters. :return: (stdout, stderr, exit code) """ logging.debug('Called run_nosetests for %s' % get_run_id(first, last)) try: # Init the outputs console = stdout = stderr = '' output_file = open_nosetests_output('log', first, last) xunit_output = open_nosetests_output(NOSE_XUNIT_EXT, first, last) # Configure the xunit output before running the command nose_cmd %= xunit_output.name except Exception as e: logging.warning('Failed to initialize run_nosetests: "%s"' % e) return logging.debug('Starting (%s): "%s"' % (get_run_id(first, last), nose_cmd)) # Start the nosetests process cmd_args = shlex.split(nose_cmd) p = subprocess.Popen(cmd_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False, universal_newlines=True, preexec_fn=os.setsid) # Read output while the process is alive idle_time = 0 select_timeout = 1 while p.poll() is None: reads, _, _ = select.select([p.stdout, p.stderr], [], [], select_timeout) for r in reads: idle_time = 0 # Write to the output file out = r.read(1) output_file.write(out) output_file.flush() console += out # Write the output to the strings if r is p.stdout: stdout += out else: stderr += out else: idle_time += select_timeout if idle_time > NOSE_TIMEOUT: # There is a special case which happens with the first call to # nose where the tests finish successfully (OK shown) but the # nosetests process doesn't end. Handle that case here: if console.strip().split('\n')[-1].startswith( 'OK') and 'Ran ' in console: msg = 'TIMEOUT after success at wrapper (%s)' stdout = add_message(msg % get_run_id(first, last), output_file, stdout) # Send the signal to all the process groups os.killpg(p.pid, signal.SIGTERM) p.returncode = 0 logging.debug('Process %s killed' % get_run_id(first, last)) break # Log everywhere I can: msg = 'TIMEOUT after error at wrapper (%s)' stdout = add_message(msg % get_run_id(first, last), output_file, stdout) args = (nose_cmd, get_run_id(first, last)) logging.warning('"%s" (%s) timeout waiting for output.' % args) # Kill the nosetests command # Send the signal to all the process groups os.killpg(p.pid, signal.SIGTERM) p.returncode = -1 logging.debug('Process %s killed' % get_run_id(first, last)) break logging.debug('Cleanup for %s' % get_run_id(first, last)) # Make sure all the output is read, there were cases when the process ended # and there were still bytes in stdout/stderr. c_stdout, c_stderr = p.communicate() stdout += c_stdout output_file.write(c_stdout) stderr += c_stderr output_file.write(c_stderr) # Close the output output_file.close() logging.debug('Finished (%s): "%s" with code "%s"' % (get_run_id(first, last), nose_cmd, p.returncode)) return nose_cmd, stdout, stderr, p.returncode, output_file.name
if __name__ == '__main__': exit_codes = [] future_list = [] done_list = [] queued_run_ids = [] start_time = time.time() configure_logging(LOG_FILE) with futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: for nose_cmd, first, last in nose_strategy(): args = run_nosetests, nose_cmd, first, last run_id = get_run_id(first, last) queued_run_ids.append(run_id) future = executor.submit(*args) future.run_id = run_id future_list.append(future) total_tests = len(future_list) print_status(start_time, done_list, total_tests, queued_run_ids, executor) while future_list: try: for future in futures.as_completed(future_list, timeout=120): try: cmd, stdout, stderr, exit_code, output_fname = future.result()
if __name__ == "__main__": exit_codes = [] future_list = [] done_list = [] queued_run_ids = [] start_time = time.time() configure_logging(LOG_FILE) with futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor: for nose_cmd, first, last in nose_strategy(): args = run_nosetests, nose_cmd, first, last run_id = get_run_id(first, last) queued_run_ids.append(run_id) future = executor.submit(*args) future.run_id = run_id future_list.append(future) total_tests = len(future_list) print_status(start_time, done_list, total_tests, queued_run_ids, executor) while future_list: try: for future in futures.as_completed(future_list, timeout=120): try: cmd, stdout, stderr, exit_code, output_fname = future.result() except Exception as e:
def run_nosetests(nose_cmd, first, last): """ Run nosetests and return the output :param nose_cmd: The nosetests command, with all parameters. :return: (stdout, stderr, exit code) """ logging.debug('Called run_nosetests for %s' % get_run_id(first, last)) try: # Init the outputs console = stdout = stderr = '' output_file = open_nosetests_output('log', first, last) xunit_output = open_nosetests_output(NOSE_XUNIT_EXT, first, last) # Configure the xunit output before running the command nose_cmd %= xunit_output.name except Exception as e: logging.warning('Failed to initialize run_nosetests: "%s"' % e) return logging.debug('Starting (%s): "%s"' % (get_run_id(first, last), nose_cmd)) # Start the nosetests process cmd_args = shlex.split(nose_cmd) p = subprocess.Popen( cmd_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False, universal_newlines=True, preexec_fn=os.setsid ) # Read output while the process is alive idle_time = 0 select_timeout = 1 while p.poll() is None: reads, _, _ = select.select([p.stdout, p.stderr], [], [], select_timeout) for r in reads: idle_time = 0 # Write to the output file out = r.read(1) output_file.write(out) output_file.flush() console += out # Write the output to the strings if r is p.stdout: stdout += out else: stderr += out else: idle_time += select_timeout if idle_time > NOSE_TIMEOUT: # There is a special case which happens with the first call to # nose where the tests finish successfully (OK shown) but the # nosetests process doesn't end. Handle that case here: if console.strip().split('\n')[-1].startswith('OK') and 'Ran ' in console: msg = 'TIMEOUT after success at wrapper (%s)' stdout = add_message(msg % get_run_id(first, last), output_file, stdout) # Send the signal to all the process groups os.killpg(p.pid, signal.SIGTERM) p.returncode = 0 logging.debug('Process %s killed' % get_run_id(first, last)) break # Log everywhere I can: msg = 'TIMEOUT after error at wrapper (%s)' stdout = add_message(msg % get_run_id(first, last), output_file, stdout) args = (nose_cmd, get_run_id(first, last)) logging.warning('"%s" (%s) timeout waiting for output.' % args) # Kill the nosetests command # Send the signal to all the process groups os.killpg(p.pid, signal.SIGTERM) p.returncode = -1 logging.debug('Process %s killed' % get_run_id(first, last)) break logging.debug('Cleanup for %s' % get_run_id(first, last)) # Make sure all the output is read, there were cases when the process ended # and there were still bytes in stdout/stderr. c_stdout, c_stderr = p.communicate() stdout += c_stdout output_file.write(c_stdout) stderr += c_stderr output_file.write(c_stderr) # Close the output output_file.close() logging.debug('Finished (%s): "%s" with code "%s"' % (get_run_id(first, last), nose_cmd, p.returncode)) return nose_cmd, stdout, stderr, p.returncode, output_file.name