Exemple #1
0
def run_nosetests(nose_cmd, first, last):
    """
    Run nosetests and return the output
    
    :param nose_cmd: The nosetests command, with all parameters.
    :return: (stdout, stderr, exit code) 
    """
    logging.debug('Called run_nosetests for %s' % get_run_id(first, last))

    try:
        # Init the outputs
        console = stdout = stderr = ''
        output_file = open_nosetests_output('log', first, last)
        xunit_output = open_nosetests_output(NOSE_XUNIT_EXT, first, last)

        # Configure the xunit output before running the command
        nose_cmd %= xunit_output.name
    except Exception as e:
        logging.warning('Failed to initialize run_nosetests: "%s"' % e)
        return

    logging.debug('Starting (%s): "%s"' % (get_run_id(first, last), nose_cmd))

    # Start the nosetests process
    cmd_args = shlex.split(nose_cmd)

    p = subprocess.Popen(cmd_args,
                         stdin=subprocess.PIPE,
                         stdout=subprocess.PIPE,
                         stderr=subprocess.PIPE,
                         shell=False,
                         universal_newlines=True,
                         preexec_fn=os.setsid)

    # Read output while the process is alive
    idle_time = 0
    select_timeout = 1

    while p.poll() is None:
        reads, _, _ = select.select([p.stdout, p.stderr], [], [],
                                    select_timeout)
        for r in reads:
            idle_time = 0
            # Write to the output file
            out = r.read(1)
            output_file.write(out)
            output_file.flush()
            console += out

            # Write the output to the strings
            if r is p.stdout:
                stdout += out
            else:
                stderr += out
        else:
            idle_time += select_timeout
            if idle_time > NOSE_TIMEOUT:
                # There is a special case which happens with the first call to
                # nose where the tests finish successfully (OK shown) but the
                # nosetests process doesn't end. Handle that case here:
                if console.strip().split('\n')[-1].startswith(
                        'OK') and 'Ran ' in console:
                    msg = 'TIMEOUT after success at wrapper (%s)'
                    stdout = add_message(msg % get_run_id(first, last),
                                         output_file, stdout)

                    # Send the signal to all the process groups
                    os.killpg(p.pid, signal.SIGTERM)
                    p.returncode = 0

                    logging.debug('Process %s killed' %
                                  get_run_id(first, last))

                    break

                # Log everywhere I can:
                msg = 'TIMEOUT after error at wrapper (%s)'
                stdout = add_message(msg % get_run_id(first, last),
                                     output_file, stdout)

                args = (nose_cmd, get_run_id(first, last))
                logging.warning('"%s" (%s) timeout waiting for output.' % args)

                # Kill the nosetests command
                # Send the signal to all the process groups
                os.killpg(p.pid, signal.SIGTERM)
                p.returncode = -1

                logging.debug('Process %s killed' % get_run_id(first, last))

                break

    logging.debug('Cleanup for %s' % get_run_id(first, last))

    # Make sure all the output is read, there were cases when the process ended
    # and there were still bytes in stdout/stderr.
    c_stdout, c_stderr = p.communicate()
    stdout += c_stdout
    output_file.write(c_stdout)

    stderr += c_stderr
    output_file.write(c_stderr)

    # Close the output
    output_file.close()

    logging.debug('Finished (%s): "%s" with code "%s"' %
                  (get_run_id(first, last), nose_cmd, p.returncode))

    return nose_cmd, stdout, stderr, p.returncode, output_file.name
Exemple #2
0

if __name__ == '__main__':
    exit_codes = []
    future_list = []
    done_list = []
    queued_run_ids = []
    start_time = time.time()

    configure_logging(LOG_FILE)

    with futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
        for nose_cmd, first, last in nose_strategy():
            args = run_nosetests, nose_cmd, first, last

            run_id = get_run_id(first, last)
            queued_run_ids.append(run_id)

            future = executor.submit(*args)
            future.run_id = run_id
            future_list.append(future)
        
        total_tests = len(future_list)
        print_status(start_time, done_list, total_tests, queued_run_ids,
                     executor)
        
        while future_list:
            try:
                for future in futures.as_completed(future_list, timeout=120):
                    try:
                        cmd, stdout, stderr, exit_code, output_fname = future.result()
Exemple #3
0

if __name__ == "__main__":
    exit_codes = []
    future_list = []
    done_list = []
    queued_run_ids = []
    start_time = time.time()

    configure_logging(LOG_FILE)

    with futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
        for nose_cmd, first, last in nose_strategy():
            args = run_nosetests, nose_cmd, first, last

            run_id = get_run_id(first, last)
            queued_run_ids.append(run_id)

            future = executor.submit(*args)
            future.run_id = run_id
            future_list.append(future)

        total_tests = len(future_list)
        print_status(start_time, done_list, total_tests, queued_run_ids, executor)

        while future_list:
            try:
                for future in futures.as_completed(future_list, timeout=120):
                    try:
                        cmd, stdout, stderr, exit_code, output_fname = future.result()
                    except Exception as e:
Exemple #4
0
def run_nosetests(nose_cmd, first, last):
    """
    Run nosetests and return the output
    
    :param nose_cmd: The nosetests command, with all parameters.
    :return: (stdout, stderr, exit code) 
    """
    logging.debug('Called run_nosetests for %s' % get_run_id(first, last))

    try:
        # Init the outputs
        console = stdout = stderr = ''
        output_file = open_nosetests_output('log', first, last)
        xunit_output = open_nosetests_output(NOSE_XUNIT_EXT, first, last)
    
        # Configure the xunit output before running the command
        nose_cmd %= xunit_output.name
    except Exception as e:
        logging.warning('Failed to initialize run_nosetests: "%s"' % e)
        return

    logging.debug('Starting (%s): "%s"' % (get_run_id(first, last), nose_cmd))

    # Start the nosetests process
    cmd_args = shlex.split(nose_cmd)

    p = subprocess.Popen(
        cmd_args,
        stdin=subprocess.PIPE,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        shell=False,
        universal_newlines=True,
        preexec_fn=os.setsid
    )
    
    # Read output while the process is alive
    idle_time = 0
    select_timeout = 1
    
    while p.poll() is None:
        reads, _, _ = select.select([p.stdout, p.stderr], [], [], select_timeout)
        for r in reads:
            idle_time = 0
            # Write to the output file
            out = r.read(1)
            output_file.write(out)
            output_file.flush()
            console += out
            
            # Write the output to the strings
            if r is p.stdout:
                stdout += out
            else:
                stderr += out
        else:
            idle_time += select_timeout
            if idle_time > NOSE_TIMEOUT:
                # There is a special case which happens with the first call to
                # nose where the tests finish successfully (OK shown) but the
                # nosetests process doesn't end. Handle that case here:
                if console.strip().split('\n')[-1].startswith('OK') and 'Ran ' in console:
                    msg = 'TIMEOUT after success at wrapper (%s)'
                    stdout = add_message(msg % get_run_id(first, last),
                                         output_file,
                                         stdout)

                    # Send the signal to all the process groups
                    os.killpg(p.pid, signal.SIGTERM)
                    p.returncode = 0

                    logging.debug('Process %s killed' % get_run_id(first, last))

                    break

                # Log everywhere I can:
                msg = 'TIMEOUT after error at wrapper (%s)'
                stdout = add_message(msg % get_run_id(first, last),
                                     output_file,
                                     stdout)

                args = (nose_cmd, get_run_id(first, last))
                logging.warning('"%s" (%s) timeout waiting for output.' % args)
                
                # Kill the nosetests command
                # Send the signal to all the process groups
                os.killpg(p.pid, signal.SIGTERM)
                p.returncode = -1

                logging.debug('Process %s killed' % get_run_id(first, last))

                break

    logging.debug('Cleanup for %s' % get_run_id(first, last))

    # Make sure all the output is read, there were cases when the process ended
    # and there were still bytes in stdout/stderr.
    c_stdout, c_stderr = p.communicate()
    stdout += c_stdout
    output_file.write(c_stdout)
    
    stderr += c_stderr
    output_file.write(c_stderr)
    
    # Close the output   
    output_file.close()
    
    logging.debug('Finished (%s): "%s" with code "%s"' % (get_run_id(first, last),
                                                          nose_cmd,
                                                          p.returncode))
    
    return nose_cmd, stdout, stderr, p.returncode, output_file.name