def run_test(
test_cmd,
pkg_path,
name,
retryable,
continue_on_errors,
):
"""Run test_cmd in the given pkg_path."""
logging.info('Running `%s` in %s...', ' '.join(test_cmd), pkg_path)
start = datetime.now()
proc = subprocess.Popen(
test_cmd,
cwd=pkg_path,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
preexec_fn=os.setsid,
shell=False)
stdout = []
def kill_proc():
logging.info('KILLING TEST: %s', name)
# Kill the group so child processes are also cleaned up.
os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
timer = threading.Timer(MAX_RUN_TIME_IN_SECONDS, kill_proc)
try:
timer.start()
for line in iter(proc.stdout.readline, ''):
# line is a bytes string literal in Python 3.
logging.info('[%s %s]: %s', test_cmd[0], name, line.rstrip().decode('utf-8'))
stdout.append(line)
proc.wait()
finally:
timer.cancel()
end = datetime.now()
if proc.returncode:
logging.info(
'TEST %s: %s (exit code: %d)\nstdout:\n%s',
'ERROR' if timer.is_alive() else 'TIMED OUT',
name,
proc.returncode,
''.join(stdout).rstrip(),
)
if retryable and is_retryable_error(''.join(stdout)):
logging.info('RETRYING TEST: %s', name)
time.sleep(3)
run_test(test_cmd, pkg_path, name, False, continue_on_errors)
return
if not continue_on_errors:
raise utils.TestFailureError(
'TEST FAILED: %s %s (exit code: %d)' %
(test_cmd[0], name, proc.returncode),
proc.returncode,
)
else:
logging.info('TEST PASSED: %s (%s seconds)', name, (end - start).seconds)
def run_test(test_cmd,
pkg_path,
name,
retryable,
continue_on_errors,
is_integration_test=False):
"""Run test_cmd in the given pkg_path."""
logging.info("Running `%s` in %s...", " ".join(test_cmd), pkg_path)
start = datetime.now()
env = None
# If we're running integration tests in parallel, assign a unique NUCLIDE_SERVER_PORT.
if is_integration_test:
process_identity = multiprocessing.current_process()._identity
if len(process_identity) > 0:
# process_id is numbered starting from 1.
process_id = process_identity[0] - 1
if process_id < len(OPEN_PORTS):
env = os.environ.copy()
env["TEST_NUCLIDE_SERVER_PORT"] = str(OPEN_PORTS[process_id])
proc = subprocess.Popen(
test_cmd,
cwd=pkg_path,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
preexec_fn=os.setsid,
shell=False,
env=env,
)
# Record the pgid now - sometimes the process exits but not its children.
proc_pgid = os.getpgid(proc.pid)
stdout = []
def kill_proc():
logging.info("KILLING TEST: %s", name)
# Kill the group so child processes are also cleaned up.
os.killpg(proc_pgid, signal.SIGKILL)
timer = threading.Timer(MAX_RUN_TIME_IN_SECONDS, kill_proc)
try:
timer.start()
for line in iter(proc.stdout.readline, ""):
# line is a bytes string literal in Python 3.
logging.info("[%s %s]: %s", test_cmd[0], name,
line.rstrip().decode("utf-8"))
stdout.append(line)
proc.wait()
# pylint: disable-msg=W0612
# flake8: noqa
except KeyboardInterrupt as e:
# Cleanly kill all child processes before terminating.
kill_proc()
sys.exit(1)
finally:
timer.cancel()
end = datetime.now()
if proc.returncode:
logging.info(
"TEST %s: %s (exit code: %d)\nstdout:\n%s",
"ERROR" if timer.is_alive() else "TIMED OUT",
name,
proc.returncode,
"".join(stdout).rstrip(),
)
if retryable and is_retryable_error("".join(stdout)):
logging.info("RETRYING TEST: %s", name)
time.sleep(3)
run_test(test_cmd, pkg_path, name, False, continue_on_errors)
return
if not continue_on_errors:
raise utils.TestFailureError(
"TEST FAILED: %s %s (exit code: %d)" %
(test_cmd[0], name, proc.returncode),
proc.returncode,
)
else:
logging.info("TEST PASSED: %s (%s seconds)", name,
(end - start).seconds)