def _run_with_nailgun(self, argv, env, java11_test_mode):
"""
Run the command using nailgun. If the daemon is busy, block until it becomes free.
"""
exit_code = 2
busy_diagnostic_displayed = False
while exit_code == 2:
try:
with NailgunConnection(
self._buck_project.get_buckd_transport_address(),
cwd=self._buck_project.root,
) as c:
now = int(round(time.time() * 1000))
env["BUCK_PYTHON_SPACE_INIT_TIME"] = str(
now - self._init_timestamp)
exit_code = c.send_command(
"com.facebook.buck.cli.Main",
self._add_args_from_env(argv),
env=env,
cwd=self._buck_project.root,
)
if exit_code == 2:
env["BUCK_BUILD_ID"] = str(uuid.uuid4())
if busy_diagnostic_displayed:
sys.stderr.write(".")
sys.stderr.flush()
else:
logging.info(
"You can use 'buck kill' to kill buck " +
"if you suspect buck is stuck.")
busy_diagnostic_displayed = True
env["BUCK_BUSY_DISPLAYED"] = "1"
sys.stderr.write(
"Waiting for Buck Daemon to become free")
sys.stderr.flush()
time.sleep(3)
except NailgunException as nex:
if nex.code == NailgunException.CONNECTION_BROKEN:
message = (
"Connection is lost to Buck daemon! This usually indicates that"
+
" daemon experienced an unrecoverable error. Here is what you can do:\n"
+
" - check if the machine has enough disk space and filesystem is"
+ " accessible\n" +
" - check if the machine does not run out of physical memory\n"
+ " - try to run Buck in serverless mode:" +
" buck kill && NO_BUCKD=1 buck <command>\n")
transport_address = self._buck_project.get_buckd_transport_address(
)
if not transport_address.startswith("local:"):
message += (" - check if connection specified by " +
transport_address + " is stable\n")
raise BuckDaemonErrorException(message)
else:
raise nex
return exit_code
def _is_buckd_running(self):
with Tracing("BuckTool._is_buckd_running"):
transport_file_path = self._buck_project.get_buckd_transport_file_path(
)
if not transport_exists(transport_file_path):
return False
try:
with NailgunConnection(
self._buck_project.get_buckd_transport_address(),
stdin=None,
stdout=None,
stderr=None,
cwd=self._buck_project.root,
) as c:
c.send_command("ng-stats")
except NailgunException as e:
if e.code in (
NailgunException.CONNECT_FAILED,
NailgunException.CONNECTION_BROKEN,
):
return False
else:
raise
return True
def tearDown(self):
try:
with NailgunConnection(
self.transport_address,
cwd=os.getcwd(),
stderr=None,
stdin=None,
stdout=None,
) as c:
c.send_command("ng-stop")
except NailgunException as e:
# stopping server is a best effort
# if something wrong has happened, we will kill it anyways
pass
# Python2 compatible wait with timeout
process_exit_code = None
for _ in range(0, 500):
process_exit_code = self.ng_server_process.poll()
if process_exit_code is not None:
break
time.sleep(0.02) # 1 second total
if process_exit_code is None:
# some test has failed, ng-server was not stopped. killing it
self.ng_server_process.kill()
debug_logs = os.environ.get("DEBUG_LOGS") or ""
if debug_logs != "":
with open(self.log_file, "r") as log_file:
print("NAILGUN SERVER LOG:\n")
print(log_file.read())
shutil.rmtree(self.tmpdir)
def kill_buckd(self):
with Tracing("BuckTool.kill_buckd"), exclusive_lock(
self._buck_project.get_section_lock_path("buckd_kill"),
wait=True):
buckd_transport_file_path = (
self._buck_project.get_buckd_transport_file_path())
wait_for_termination = False
if transport_exists(buckd_transport_file_path):
logging.debug("Shutting down buck daemon.")
try:
with NailgunConnection(
self._buck_project.get_buckd_transport_address(),
cwd=self._buck_project.root,
) as c:
c.send_command("ng-stop")
wait_for_termination = True
except NailgunException as e:
if e.code not in (
NailgunException.CONNECT_FAILED,
NailgunException.CONNECTION_BROKEN,
NailgunException.UNEXPECTED_CHUNKTYPE,
):
raise BuckToolException(
"Unexpected error shutting down nailgun server: " +
str(e))
if os.name == "posix":
buckd_pid = self._buck_project.get_running_buckd_pid()
if pid_exists_posix(buckd_pid):
# If termination command was sent successfully then allow some time for the
# daemon to finish gracefully. Otherwise kill it hard.
if not wait_for_termination or not wait_for_process_posix(
buckd_pid, 5000):
# There is a possibility that daemon is dead for some time but pid file
# still exists and another process is assigned to the same pid. Ideally we
# should check first which process we are killing but so far let's pretend
# this will never happen and kill it with fire anyways.
try:
force_kill_process_posix(buckd_pid)
except Exception as e:
# In the worst case it keeps running multiple daemons simultaneously
# consuming memory. Not the good place to be, but let's just issue a
# warning for now.
logging.warning(
"Error killing running Buck daemon " + str(e))
# it is ok to have a socket file still around, as linux domain sockets should be
# unlinked explicitly
if transport_exists(buckd_transport_file_path):
force_close_transport_posix(buckd_transport_file_path)
elif os.name == "nt":
# for Windows, we rely on transport to be closed to determine the process is done
# TODO(buck_team) implement wait for process and hard kill for Windows
for _idx in range(0, 300):
if not transport_exists(buckd_transport_file_path):
break
time.sleep(0.01)
self._buck_project.clean_up_buckd()
def test_nailgun_exit_code(self):
output = StringIO()
expected_exit_code = 10
with NailgunConnection(
self.transport_address, stderr=None, stdin=None, stdout=output
) as c:
exit_code = c.send_command(
"com.facebook.nailgun.examples.Exit", [str(expected_exit_code)]
)
self.assertEqual(exit_code, expected_exit_code)
def test_nailgun_stats(self):
output = StringIO()
with NailgunConnection(
self.transport_address, stderr=None, stdin=None, stdout=output
) as c:
exit_code = c.send_command("ng-stats")
self.assertEqual(exit_code, 0)
actual_out = output.getvalue().strip()
expected_out = "com.facebook.nailgun.builtins.NGServerStats: 1/1"
self.assertEqual(actual_out, expected_out)
def kill_buckd(self):
with Tracing("BuckTool.kill_buckd"):
buckd_transport_file_path = (
self._buck_project.get_buckd_transport_file_path())
if transport_exists(buckd_transport_file_path):
buckd_pid = self._buck_project.get_running_buckd_pid()
logging.debug("Shutting down buck daemon.")
wait_socket_close = False
try:
with NailgunConnection(
self._buck_project.get_buckd_transport_address(),
cwd=self._buck_project.root,
) as c:
c.send_command("ng-stop")
wait_socket_close = True
except NailgunException as e:
if e.code not in (
NailgunException.CONNECT_FAILED,
NailgunException.CONNECTION_BROKEN,
NailgunException.UNEXPECTED_CHUNKTYPE,
):
raise BuckToolException(
"Unexpected error shutting down nailgun server: " +
str(e))
# If ng-stop command succeeds, wait for buckd process to terminate and for the
# socket to close. On Unix ng-stop always drops the connection and throws.
if wait_socket_close:
for i in range(0, 300):
if not transport_exists(buckd_transport_file_path):
break
time.sleep(0.01)
elif buckd_pid is not None and os.name == "posix":
# otherwise just wait for up to 5 secs for the process to die
# TODO(buck_team) implement wait for process and hard kill for Windows too
if not wait_for_process_posix(buckd_pid, 5000):
# There is a possibility that daemon is dead for some time but pid file
# still exists and another process is assigned to the same pid. Ideally we
# should check first which process we are killing but so far let's pretend
# this will never happen and kill it with fire anyways.
try:
force_kill_process_posix(buckd_pid)
except Exception as e:
# In the worst case it keeps running multiple daemons simultaneously
# consuming memory. Not the good place to be, but let's just issue a
# warning for now.
logging.warning(
"Error killing running Buck daemon " + str(e))
if transport_exists(buckd_transport_file_path):
force_close_transport(buckd_transport_file_path)
self._buck_project.clean_up_buckd()
def test_nailgun_handles_nonutf8_on_stdout(self):
output = BytesIO()
with NailgunConnection(self.transport_address,
stderr=None,
stdin=None,
stdout=output) as c:
exit_code = c.send_command(
"com.facebook.nailgun.examples.BinaryEcho", ["2048"])
self.assertEqual(exit_code, 0)
actual_out = output.getvalue()
self.assertEqual(actual_out, b"\xe2" * 2048)
def test_nailgun_no_heartbeat(self):
output = StringIO()
with NailgunConnection(
self.transport_address,
stderr=None,
stdin=None,
stdout=output,
heartbeat_interval_sec=0,
) as c:
exit_code = c.send_command(
"com.martiansoftware.nailgun.examples.Heartbeat", ["3000"])
self.assertTrue(output.getvalue().count("H") == 0)
def test_nailgun_stdin(self):
lines = [str(i) for i in range(100)]
echo = "\n".join(lines)
output = StringIO()
input = StringIO(echo)
with NailgunConnection(
self.transport_address, stderr=None, stdin=input, stdout=output
) as c:
exit_code = c.send_command("com.facebook.nailgun.examples.Echo")
self.assertEqual(exit_code, 0)
actual_out = output.getvalue().strip()
self.assertEqual(actual_out, echo)
def test_stress_nailgun_socket_close_without_race_condition(self):
output = StringIO()
for i in range(1000):
with NailgunConnection(
self.transport_address,
stderr=None,
stdin=None,
stdout=output,
heartbeat_interval_sec=0.001,
) as c:
exit_code = c.send_command(
"com.martiansoftware.nailgun.examples.Heartbeat", ["10"])
self.assertEqual(exit_code, 0)
def test_nailgun_heartbeats(self):
output = StringIO()
with NailgunConnection(
self.transport_address,
stderr=None,
stdin=None,
stdout=output,
heartbeat_interval_sec=0.1,
) as c:
# just run Heartbeat nail for 5 seconds. During this period there should be
# heartbeats received and printed back
exit_code = c.send_command(
"com.martiansoftware.nailgun.examples.Heartbeat", ["5000"])
self.assertTrue(output.getvalue().count("H") > 10)
def test_nailgun_with_utf8_environ(self):
output = BytesIO()
with NailgunConnection(self.transport_address,
stderr=None,
stdin=None,
stdout=output) as c:
env = os.environ.copy()
# foo <BOX DRAWINGS LIGHT DOWN AND RIGHT> <BOX DRAWINGS LIGHT HORIZONTAL> bar
env["WITH_UTF8"] = "foo\xe2\x94\x8c\xe2\x94\x80bar"
exit_code = c.send_command("ng-stats", env=env)
self.assertEqual(exit_code, 0)
actual_out = output.getvalue().decode('utf8').strip()
expected_out = "com.facebook.nailgun.builtins.NGServerStats: 1/1"
self.assertEqual(actual_out, expected_out)
def test_nailgun_disconnect(self):
"""
We should disconnect before time elapses because of configuration:
Heartbeats are sent every 5 secs
Server expects to look for disconnects if no hearbeat is received in 1 sec
Server runs for 30 sec given we still have heartbeats, so it should output about 6 'H'
We assert that number of 'H' is smaller
"""
output = StringIO()
with NailgunConnection(
self.transport_address,
stderr=None,
stdin=None,
stdout=output,
heartbeat_interval_sec=5,
) as c:
exit_code = c.send_command(
"com.martiansoftware.nailgun.examples.Heartbeat", ["30000"])
self.assertTrue(output.getvalue().count("H") < 3)
def test_nailgun_default_streams(self):
with NailgunConnection(self.transport_address) as c:
exit_code = c.send_command("ng-stats")
self.assertEqual(exit_code, 0)
