def launch_buck(self, build_id):
with Tracing('BuckTool.launch_buck'):
with JvmCrashLogger(self, self._buck_project.root):
if self._command_line.command == "clean" and \
not self._command_line.is_help():
self.kill_buckd()
buck_version_uid = self._get_buck_version_uid()
if self._command_line.is_version():
print("buck version {}".format(buck_version_uid))
return 0
use_buckd = self._use_buckd
if not self._command_line.is_help():
has_watchman = bool(which('watchman'))
if use_buckd and has_watchman:
running_version = self._buck_project.get_running_buckd_version()
if running_version != buck_version_uid:
self.kill_buckd()
if not self._is_buckd_running():
self.launch_buckd(buck_version_uid=buck_version_uid)
elif use_buckd and not has_watchman:
logging.warning("Not using buckd because watchman isn't installed.")
elif not use_buckd:
logging.warning("Not using buckd because NO_BUCKD is set.")
env = self._environ_for_buck()
env['BUCK_BUILD_ID'] = build_id
if use_buckd and self._is_buckd_running():
return self._run_with_buckd(env)
command = ["buck"]
extra_default_options = [
"-Djava.io.tmpdir={0}".format(self._tmp_dir),
"-Dfile.encoding=UTF-8",
"-XX:SoftRefLRUPolicyMSPerMB=0",
"-XX:+UseG1GC",
]
command.extend(self._get_java_args(buck_version_uid, extra_default_options))
command.append("com.facebook.buck.cli.bootstrapper.ClassLoaderBootstrapper")
command.append("com.facebook.buck.cli.Main")
command.extend(sys.argv[1:])
now = int(round(time.time() * 1000))
env['BUCK_PYTHON_SPACE_INIT_TIME'] = str(now - self._init_timestamp)
if True:
java = which("java")
if java is None:
raise BuckToolException('Could not find java on $PATH')
with Tracing('buck', args={'command': command}):
buck_exit_code = subprocess.call(command,
cwd=self._buck_project.root,
env=env,
executable=java)
return buck_exit_code
def launch_buck(self, build_id):
with Tracing('BuckTool.launch_buck'):
with JvmCrashLogger(self, self._buck_project.root):
if self._command_line.command == "clean" and \
not self._command_line.is_help():
self.kill_buckd()
buck_version_uid = self._get_buck_version_uid()
if self._command_line.is_version():
print("buck version {}".format(buck_version_uid))
return 0
use_buckd = self._use_buckd
if not self._command_line.is_help():
has_watchman = bool(which('watchman'))
if use_buckd and has_watchman:
running_version = self._buck_project.get_running_buckd_version()
if running_version != buck_version_uid:
self.kill_buckd()
if not self._is_buckd_running():
self.launch_buckd(buck_version_uid=buck_version_uid)
elif use_buckd and not has_watchman:
logging.warning("Not using buckd because watchman isn't installed.")
elif not use_buckd:
logging.warning("Not using buckd because NO_BUCKD is set.")
env = self._environ_for_buck()
env['BUCK_BUILD_ID'] = build_id
if use_buckd and self._is_buckd_running():
return self._run_with_buckd(env)
command = ["buck"]
extra_default_options = [
"-Djava.io.tmpdir={0}".format(self._tmp_dir),
"-Dfile.encoding=UTF-8",
"-XX:SoftRefLRUPolicyMSPerMB=0",
"-XX:+UseG1GC",
]
command.extend(self._get_java_args(buck_version_uid, extra_default_options))
command.append("com.facebook.buck.cli.bootstrapper.ClassLoaderBootstrapper")
command.append("com.facebook.buck.cli.Main")
command.extend(sys.argv[1:])
now = int(round(time.time() * 1000))
env['BUCK_PYTHON_SPACE_INIT_TIME'] = str(now - self._init_timestamp)
if True:
java = which("java")
if java is None:
raise BuckToolException('Could not find java on $PATH')
with Tracing('buck', args={'command': command}):
buck_exit_code = subprocess.call(command,
cwd=self._buck_project.root,
env=env,
executable=java)
return buck_exit_code
def __init__(self, buck_bin_dir, buck_project, buck_reporter):
self.buck_dir = platform_path(os.path.dirname(buck_bin_dir))
super(BuckRepo, self).__init__(buck_project, buck_reporter)
dot_git = os.path.join(self.buck_dir, ".git")
self.is_git = (
os.path.exists(dot_git)
and os.path.isdir(dot_git)
and which("git")
and sys.platform != "cygwin"
)
self._is_buck_repo_dirty_override = os.environ.get("BUCK_REPOSITORY_DIRTY")
if not self._fake_buck_version:
# self._fake_buck_version has been set previously through BuckTool when the environment
# variable BUCK_FAKE_VERSION is set.
# If the environement variable is not set, we'll use the content of .fakebuckversion
# at the root of the repository if it exists.
fake_buck_version_file_path = os.path.join(
self.buck_dir, ".fakebuckversion"
)
if os.path.exists(fake_buck_version_file_path):
with open(fake_buck_version_file_path) as fake_buck_version_file:
self._fake_buck_version = fake_buck_version_file.read().strip()
logging.info(
"Using fake buck version (via .fakebuckversion): {}".format(
self._fake_buck_version
)
)
def _run_without_nailgun(self, argv, env):
'''
Run the command by directly invoking `java` (rather than by sending a command via nailgun)
'''
command = ["buck"]
extra_default_options = [
"-Djava.io.tmpdir={0}".format(self._tmp_dir),
"-Dfile.encoding=UTF-8",
"-XX:SoftRefLRUPolicyMSPerMB=0",
"-XX:+UseG1GC",
]
command.extend(
self._get_java_args(self._get_buck_version_uid(),
extra_default_options))
command.append(
"com.facebook.buck.cli.bootstrapper.ClassLoaderBootstrapper")
command.append("com.facebook.buck.cli.Main")
command.extend(argv)
now = int(round(time.time() * 1000))
env['BUCK_PYTHON_SPACE_INIT_TIME'] = str(now - self._init_timestamp)
java = which('java')
if java is None:
raise BuckToolException('Could not find java on $PATH')
with Tracing('buck', args={'command': command}):
return subprocess.call(command,
cwd=self._buck_project.root,
env=env,
executable=java)
def _run_without_nailgun(self, argv, env):
"""
Run the command by directly invoking `java` (rather than by sending a command via nailgun)
"""
command = ["buck"]
extra_default_options = [
"-Djava.io.tmpdir={0}".format(self._tmp_dir),
"-Dfile.encoding=UTF-8",
"-XX:SoftRefLRUPolicyMSPerMB=0",
"-XX:+UseG1GC",
]
command.extend(self._get_java_args(self._get_buck_version_uid(), extra_default_options))
command.append("com.facebook.buck.cli.bootstrapper.ClassLoaderBootstrapper")
command.append("com.facebook.buck.cli.Main")
command.extend(self._add_args_from_env(argv))
now = int(round(time.time() * 1000))
env['BUCK_PYTHON_SPACE_INIT_TIME'] = str(now - self._init_timestamp)
java = which('java')
if java is None:
raise BuckToolException('Could not find java on $PATH')
with Tracing('buck', args={'command': command}):
return subprocess.call(command,
cwd=self._buck_project.root,
env=env,
executable=java)
def __init__(self, buck_bin_dir, buck_project):
super(BuckRepo, self).__init__(buck_project)
self.buck_dir = platform_path(os.path.dirname(buck_bin_dir))
dot_git = os.path.join(self.buck_dir, '.git')
self.is_git = os.path.exists(dot_git) and os.path.isdir(dot_git) and which('git') and \
sys.platform != 'cygwin'
self._is_buck_repo_dirty_override = os.environ.get(
'BUCK_REPOSITORY_DIRTY')
if not self._fake_buck_version:
# self._fake_buck_version has been set previously through BuckTool when the environment
# variable BUCK_FAKE_VERSION is set.
# If the environement variable is not set, we'll use the content of .fakebuckversion
# at the root of the repository if it exists.
fake_buck_version_file_path = os.path.join(self.buck_dir,
".fakebuckversion")
if os.path.exists(fake_buck_version_file_path):
with open(
fake_buck_version_file_path) as fake_buck_version_file:
self._fake_buck_version = fake_buck_version_file.read(
).strip()
logging.info(
"Using fake buck version (via .fakebuckversion): {}".
format(self._fake_buck_version))
def launch_buck(self, build_id):
with Tracing('BuckTool.launch_buck'):
with JvmCrashLogger(self, self._buck_project.root):
self._reporter.build_id = build_id
try:
repository = self._get_repository()
self._reporter.repository = repository
except Exception as e:
# _get_repository() is only for reporting,
# so skip on error
logging.warning('Failed to get repo name: ' + str(e))
if self._command_line.command == "clean" and \
not self._command_line.is_help():
self.kill_buckd()
buck_version_uid = self._get_buck_version_uid()
self._reporter.buck_version = buck_version_uid
if self._command_line.is_version():
print("buck version {}".format(buck_version_uid))
return 0
use_buckd = self._use_buckd
if not self._command_line.is_help():
has_watchman = bool(which('watchman'))
if use_buckd and has_watchman:
running_version = self._buck_project.get_running_buckd_version()
if running_version != buck_version_uid:
self.kill_buckd()
if not self._is_buckd_running():
self.launch_buckd(buck_version_uid=buck_version_uid)
elif use_buckd and not has_watchman:
logging.warning("Not using buckd because watchman isn't installed.")
elif not use_buckd:
logging.warning("Not using buckd because NO_BUCKD is set.")
env = self._environ_for_buck()
env['BUCK_BUILD_ID'] = build_id
use_nailgun = use_buckd and self._is_buckd_running()
self._reporter.is_buckd = use_nailgun
run_fn = self._run_with_nailgun if use_nailgun else self._run_without_nailgun
self._unpack_modules()
exit_code = self._execute_command_and_maybe_run_target(
run_fn, env)
# Most shells return process termination with signal as
# 128 + N, where N is the signal. However Python's subprocess
# call returns them as negative numbers. Buck binary protocol
# uses shell's convention, so convert
if (exit_code < 0):
exit_code = 128 + (-1 * exit_code)
return exit_code
def _check_for_ant(self):
ant = which('ant')
if not ant:
message = "You do not have ant on your $PATH. Cannot build Buck."
if sys.platform == "darwin":
message += "\nTry running 'brew install ant'."
raise BuckToolException(message)
return ant
def is_git(dirpath):
dot_git = os.path.join(dirpath, ".git")
if which("git") and sys.platform != "cygwin":
if os.path.exists(dot_git) and os.path.isdir(dot_git):
return True
output = check_output(["git", "rev-parse", "--is-inside-work-tree"], cwd=dirpath)
return output.strip() == "true"
return False
def find_ant():
ant = which('ant')
if not ant:
message = "You do not have ant on your $PATH. Cannot build Buck."
if sys.platform == "darwin":
message += "\nTry running 'brew install ant'."
raise RuntimeError(message)
return ant
def is_git(dirpath):
dot_git = os.path.join(dirpath, '.git')
return all([
os.path.exists(dot_git),
os.path.isdir(dot_git),
which('git'),
sys.platform != 'cygwin',
])
def is_git(dirpath):
dot_git = os.path.join(dirpath, '.git')
return all([
os.path.exists(dot_git),
os.path.isdir(dot_git),
which('git'),
sys.platform != 'cygwin',
])
def find_ant():
ant = which('ant')
if not ant:
message = "You do not have ant on your $PATH. Cannot build Buck."
if sys.platform == "darwin":
message += "\nTry running 'brew install ant'."
raise RuntimeError(message)
return ant
def _check_for_ant(self):
ant = which('ant')
if not ant:
message = "You do not have ant on your $PATH. Cannot build Buck."
if sys.platform == "darwin":
message += "\nTry running 'brew install ant'."
raise BuckToolException(message)
return ant
def launch_buck(self, build_id):
with Tracing("BuckRepo.launch_buck"):
if self._command_line.command == "clean" and not self._command_line.is_help():
self.kill_buckd()
buck_version_uid = self._get_buck_version_uid()
use_buckd = self._use_buckd()
if not self._command_line.is_help():
has_watchman = bool(which("watchman"))
if use_buckd and has_watchman:
buckd_run_count = self._buck_project.get_buckd_run_count()
running_version = self._buck_project.get_running_buckd_version()
new_buckd_run_count = buckd_run_count + 1
if buckd_run_count == MAX_BUCKD_RUN_COUNT or running_version != buck_version_uid:
self.kill_buckd()
new_buckd_run_count = 0
if new_buckd_run_count == 0 or not self._is_buckd_running():
self.launch_buckd(buck_version_uid=buck_version_uid)
else:
self._buck_project.update_buckd_run_count(new_buckd_run_count)
elif use_buckd and not has_watchman:
print("Not using buckd because watchman isn't installed.", file=sys.stderr)
elif not use_buckd:
print("Not using buckd because NO_BUCKD is set.", file=sys.stderr)
env = self._environ_for_buck()
env["BUCK_BUILD_ID"] = build_id
buck_socket_path = self._buck_project.get_buckd_socket_path()
if use_buckd and self._is_buckd_running() and os.path.exists(buck_socket_path):
with Tracing("buck", args={"command": sys.argv[1:]}):
exit_code = 2
last_diagnostic_time = 0
while exit_code == 2:
with NailgunConnection("local:.buckd/sock", cwd=self._buck_project.root) as c:
exit_code = c.send_command(
"com.facebook.buck.cli.Main", sys.argv[1:], env=env, cwd=self._buck_project.root
)
if exit_code == 2:
now = time.time()
if now - last_diagnostic_time > DAEMON_BUSY_MESSAGE_SECONDS:
print("Daemon is busy, waiting for it to become free...", file=sys.stderr)
last_diagnostic_time = now
time.sleep(1)
return exit_code
command = ["buck"]
extra_default_options = ["-Djava.io.tmpdir={0}".format(self._tmp_dir)]
command.extend(self._get_java_args(buck_version_uid, extra_default_options))
command.append("com.facebook.buck.cli.bootstrapper.ClassLoaderBootstrapper")
command.append("com.facebook.buck.cli.Main")
command.extend(sys.argv[1:])
return subprocess.call(command, cwd=self._buck_project.root, env=env, executable=which("java"))
def __init__(self, buck_bin_dir, buck_project):
super(BuckRepo, self).__init__(buck_project)
self.buck_dir = platform_path(os.path.dirname(buck_bin_dir))
dot_git = os.path.join(self.buck_dir, '.git')
self.is_git = os.path.exists(dot_git) and os.path.isdir(dot_git) and which('git') and \
sys.platform != 'cygwin'
self._is_buck_repo_dirty_override = os.environ.get('BUCK_REPOSITORY_DIRTY')
def is_git(dirpath):
dot_git = os.path.join(dirpath, '.git')
if(which('git') and sys.platform != 'cygwin'):
if(os.path.exists(dot_git) and os.path.isdir(dot_git)):
return True
output = check_output(
['git', 'rev-parse', '--is-inside-work-tree'],
cwd=dirpath)
return output.strip() == 'true'
return False
def test_java_home_for_wrong_version_ignored(self):
os.environ["JAVA_HOME"] = (
"/Library/Java/JavaVirtualMachines/jdk-" +
str(JAVA_VERSION_THAT_OBVIOUSLY_CANT_EXIST_LOCALLY + 1) +
".jdk/Contents/Home")
self.assertEquals(
get_java_path(
JAVA_VERSION_THAT_OBVIOUSLY_CANT_EXIST_LOCALLY).lower(),
which("java").lower(),
)
def is_git(dirpath):
dot_git = os.path.join(dirpath, '.git')
if which('git') and sys.platform != 'cygwin':
if os.path.exists(dot_git) and os.path.isdir(dot_git):
return True
try:
output = check_output(
['git', 'rev-parse', '--is-inside-work-tree'], cwd=dirpath)
return output.strip() == 'true'
except CalledProcessError:
pass
return False
def setup_watchman_watch():
with Tracing('BuckTool._setup_watchman_watch'):
if not which('watchman'):
message = textwrap.dedent("""\
Watchman not found, please install when using buckd.
See https://github.com/facebook/watchman for details.""")
if sys.platform == "darwin":
message += "\n(brew install watchman on OS X)"
# Bail if watchman isn't installed as we know java's
# FileSystemWatcher will take too long to process events.
raise BuckToolException(message)
logging.debug("Using watchman.")
def setup_watchman_watch():
with Tracing("BuckTool._setup_watchman_watch"):
if not which("watchman"):
message = textwrap.dedent("""\
Watchman not found, please install when using buckd.
See https://github.com/facebook/watchman for details.""")
if sys.platform == "darwin":
message += "\n(brew install watchman on OS X)"
# Bail if watchman isn't installed as we know java's
# FileSystemWatcher will take too long to process events.
raise BuckToolException(message)
logging.debug("Using watchman.")
def is_vcs(dirpath): # type: (str) -> bool
dot_hg = os.path.join(dirpath, ".hg")
if which("hg") and sys.platform != "cygwin":
if os.path.exists(dot_hg) and os.path.isdir(dot_hg):
return True
try:
with open(os.devnull, "w") as devnull:
output = check_output(
["hg", "root"], cwd=dirpath, stderr=devnull
).decode("utf-8")
return os.path.abspath(output.strip()) == os.path.abspath(dirpath)
except CalledProcessError:
pass
return False
def is_git(dirpath):
dot_git = os.path.join(dirpath, '.git')
if which('git') and sys.platform != 'cygwin':
if os.path.exists(dot_git) and os.path.isdir(dot_git):
return True
try:
with open(os.devnull, 'w') as devnull:
output = check_output(
['git', 'rev-parse', '--is-inside-work-tree'],
cwd=dirpath,
stderr=devnull)
return output.strip() == 'true'
except CalledProcessError:
pass
return False
def is_git(dirpath): # type: (str) -> bool
dot_git = os.path.join(dirpath, '.git')
if which('git') and sys.platform != 'cygwin':
if os.path.exists(dot_git) and os.path.isdir(dot_git):
return True
try:
with open(os.devnull, 'w') as devnull:
output = check_output(
['git', 'rev-parse', '--is-inside-work-tree'],
cwd=dirpath,
stderr=devnull)
return output.strip() == 'true'
except CalledProcessError:
pass
return False
def is_git(dirpath): # type: (str) -> bool
dot_git = os.path.join(dirpath, ".git")
if which("git") and sys.platform != "cygwin":
if os.path.exists(dot_git) and os.path.isdir(dot_git):
return True
try:
with open(os.devnull, "w") as devnull:
output = check_output(
["git", "rev-parse", "--is-inside-work-tree"],
cwd=dirpath,
stderr=devnull,
).decode("utf-8")
return output.strip() == "true"
except CalledProcessError:
pass
return False
def is_git(dirpath): # type: (str) -> bool
dot_git = os.path.join(dirpath, ".git")
if which("git") and sys.platform != "cygwin":
if os.path.exists(dot_git) and os.path.isdir(dot_git):
return True
try:
with open(os.devnull, "w") as devnull:
output = check_output(
["git", "rev-parse", "--is-inside-work-tree"],
cwd=dirpath,
stderr=devnull,
)
return output.strip() == "true"
except CalledProcessError:
pass
return False
def get_java_path():
java_home_path = os.getenv("JAVA_HOME")
if java_home_path is None:
java_path = which("java")
if java_path is None:
raise BuckToolException("Could not find Java executable. \
Make sure it is on PATH or JAVA_HOME is set.")
else:
java_path = _get_java_exec_under_home(java_home_path)
if not os.path.isfile(java_path):
message = textwrap.dedent("""
Could not find Java executable under JAVA_HOME at: '{}'.
Please make sure your JAVA_HOME environment variable is set correctly.
""").format(java_path)
raise BuckToolException(message)
return java_path
def get_java_path():
java_home_path = os.getenv('JAVA_HOME')
java_path = None
if java_home_path is None:
java_path = which('java')
if java_path is None:
raise BuckToolException('Could not find Java executable. \
Make sure it is on PATH or JAVA_HOME is set.')
else:
java_path = _get_java_exec_under_home(java_home_path)
if not os.path.isfile(java_path):
message = textwrap.dedent("""
Could not find Java executable under JAVA_HOME at: '{}'.
Please make sure your JAVA_HOME environment variable is set correctly.
""").format(java_path)
raise BuckToolException(message)
return java_path
def get_java_path(required_java_version):
java_home_path = os.getenv("JAVA_HOME")
if java_home_path:
# Though we try to respect JAVA_HOME, if the path looks like the wrong version of Java, try
# to use a known location of the JDK for the right version instead.
suspected_java_version = _get_suspected_java_version_from_java_path(
java_home_path
)
if suspected_java_version and suspected_java_version != required_java_version:
message = (
'Warning: JAVA_HOME is set to "{}", which looks like a Java {} path, '
+ "but Buck requires Java {}."
).format(java_home_path, suspected_java_version, required_java_version)
if os.getenv("BUCK_RESPECT_JAVA_HOME") != "1":
message += " Ignoring JAVA_HOME. Set BUCK_RESPECT_JAVA_HOME to 1 to disable this behavior."
java_home_path = None
logging.warning(message)
if java_home_path is None:
# Default to a known location of the JDK for the right version of Java, regardless of what
# version of Java is on the PATH.
java_base_path = _get_known_java_path_for_version(required_java_version)
java_path = None
if java_base_path:
java_path = _get_java_exec(java_base_path)
if not os.path.isfile(java_path):
java_path = None
if not java_path:
java_path = which("java")
if java_path is None:
raise BuckToolException(
"Could not find Java executable. \
Make sure it is on PATH or JAVA_HOME is set."
)
else:
java_path = _get_java_exec(java_home_path)
if not os.path.isfile(java_path):
message = textwrap.dedent(
"""
Could not find Java executable under JAVA_HOME at: '{}'.
Please make sure your JAVA_HOME environment variable is set correctly.
Then restart buck (buck kill) and try again.
"""
).format(java_path)
raise BuckToolException(message)
return java_path
def __init__(self, buck_bin_dir, buck_project):
super(BuckRepo, self).__init__(buck_project)
self._buck_dir = self._platform_path(os.path.dirname(buck_bin_dir))
self._build_success_file = os.path.join(
self._buck_dir, "build", "successful-build")
dot_git = os.path.join(self._buck_dir, '.git')
self._is_git = os.path.exists(dot_git) and os.path.isdir(dot_git) and which('git') and \
sys.platform != 'cygwin'
self._is_buck_repo_dirty_override = os.environ.get('BUCK_REPOSITORY_DIRTY')
buck_version = buck_project.buck_version
if self._is_git and not buck_project.has_no_buck_check and buck_version:
revision = buck_version[0]
branch = buck_version[1] if len(buck_version) > 1 else None
self._checkout_and_clean(revision, branch)
self._build()
def launch_buck(self, build_id):
with Tracing('BuckTool.launch_buck'):
with JvmCrashLogger(self, self._buck_project.root):
if self._command_line.command == "clean" and \
not self._command_line.is_help():
self.kill_buckd()
buck_version_uid = self._get_buck_version_uid()
if self._command_line.is_version():
print("buck version {}".format(buck_version_uid))
return 0
use_buckd = self._use_buckd
if not self._command_line.is_help():
has_watchman = bool(which('watchman'))
if use_buckd and has_watchman:
running_version = self._buck_project.get_running_buckd_version(
)
if running_version != buck_version_uid:
self.kill_buckd()
if not self._is_buckd_running():
self.launch_buckd(
buck_version_uid=buck_version_uid)
elif use_buckd and not has_watchman:
logging.warning(
"Not using buckd because watchman isn't installed."
)
elif not use_buckd:
logging.warning(
"Not using buckd because NO_BUCKD is set.")
env = self._environ_for_buck()
env['BUCK_BUILD_ID'] = build_id
use_nailgun = use_buckd and self._is_buckd_running()
run_fn = self._run_with_nailgun if use_nailgun else self._run_without_nailgun
self._unpack_modules()
return self._execute_command_and_maybe_run_target(run_fn, env)
def __init__(self, buck_bin_dir, buck_project):
super(BuckRepo, self).__init__(buck_project)
self._buck_dir = self._platform_path(os.path.dirname(buck_bin_dir))
self._build_success_file = os.path.join(
self._buck_dir, "build", "successful-build")
dot_git = os.path.join(self._buck_dir, '.git')
self._is_git = os.path.exists(dot_git) and os.path.isdir(dot_git) and which('git') and \
sys.platform != 'cygwin'
self._is_buck_repo_dirty_override = os.environ.get('BUCK_REPOSITORY_DIRTY')
buck_version = buck_project.buck_version
if self._is_git and not buck_project.has_no_buck_check and buck_version:
revision = buck_version[0]
branch = buck_version[1] if len(buck_version) > 1 else None
self._checkout_and_clean(revision, branch)
self._build()
def get_java_path(required_java_version):
java_home_path = os.getenv("JAVA_HOME")
if java_home_path:
# Though we try to respect JAVA_HOME, if the path looks like the wrong version of Java, try
# to use a known location of the JDK for the right version instead.
suspected_java_version = _get_suspected_java_version_from_java_path(
java_home_path)
if suspected_java_version and suspected_java_version != required_java_version:
message = (
'Warning: JAVA_HOME is set to "{}", which looks like a Java {} path, '
+ "but Buck requires Java {}.").format(java_home_path,
suspected_java_version,
required_java_version)
if os.getenv("BUCK_RESPECT_JAVA_HOME") != "1":
message += " Ignoring JAVA_HOME. Set BUCK_RESPECT_JAVA_HOME to 1 to disable this behavior."
java_home_path = None
logging.warning(message)
if java_home_path is None:
# Default to a known location of the JDK for the right version of Java, regardless of what
# version of Java is on the PATH.
java_base_path = _get_known_java_path_for_version(
required_java_version)
java_path = None
if java_base_path:
java_path = _get_java_exec(java_base_path)
if not os.path.isfile(java_path):
java_path = None
if not java_path:
java_path = which("java")
if java_path is None:
raise BuckToolException("Could not find Java executable. \
Make sure it is on PATH or JAVA_HOME is set.")
else:
java_path = _get_java_exec(java_home_path)
if not os.path.isfile(java_path):
message = textwrap.dedent("""
Could not find Java executable under JAVA_HOME at: '{}'.
Please make sure your JAVA_HOME environment variable is set correctly.
Then restart buck (buck kill) and try again.
""").format(java_path)
raise BuckToolException(message)
return java_path
def get_java_path():
java_home_path = os.getenv("JAVA_HOME")
if java_home_path is None:
java_path = which("java")
if java_path is None:
raise BuckToolException(
"Could not find Java executable. \
Make sure it is on PATH or JAVA_HOME is set."
)
else:
java_path = _get_java_exec_under_home(java_home_path)
if not os.path.isfile(java_path):
message = textwrap.dedent(
"""
Could not find Java executable under JAVA_HOME at: '{}'.
Please make sure your JAVA_HOME environment variable is set correctly.
Then restart buck (buck kill) and try again.
"""
).format(java_path)
raise BuckToolException(message)
return java_path
def test_without_java_home(self):
self.assertEquals(
get_java_path(
JAVA_VERSION_THAT_OBVIOUSLY_CANT_EXIST_LOCALLY).lower(),
which("java").lower(),
)
def launch_buck(self, build_id, argv, java11_test_mode):
with Tracing("BuckTool.launch_buck"):
with JvmCrashLogger(self, self._buck_project.root):
self._reporter.build_id = build_id
try:
repository = self._get_repository()
self._reporter.repository = repository
except Exception as e:
# _get_repository() is only for reporting,
# so skip on error
logging.warning("Failed to get repo name: " + str(e))
if (self._command_line.command == "clean"
and not self._command_line.is_help()):
self.kill_buckd()
buck_version_uid = self._get_buck_version_uid()
self._reporter.buck_version = buck_version_uid
if self._command_line.is_version():
print("buck version {}".format(buck_version_uid))
return 0
use_buckd = self._use_buckd
if not use_buckd:
logging.warning("Not using buckd because NO_BUCKD is set.")
self._reporter.no_buckd_reason = "explicit"
if use_buckd and self._command_line.is_help():
use_buckd = False
self._reporter.no_buckd_reason = "help"
if use_buckd:
has_watchman = bool(which("watchman"))
if not has_watchman:
use_buckd = False
self._reporter.no_buckd_reason = "watchman"
logging.warning(
"Not using buckd because watchman isn't installed."
)
if use_buckd:
need_start = True
running_version = self._buck_project.get_running_buckd_version(
)
if running_version is None:
logging.info("Starting new Buck daemon...")
elif running_version != buck_version_uid:
logging.info(
"Restarting Buck daemon because Buck version has changed..."
)
elif not self._is_buckd_running():
logging.info(
"Unable to connect to Buck daemon, restarting it..."
)
else:
need_start = False
if need_start:
self.kill_buckd()
if not self.launch_buckd(
java11_test_mode,
buck_version_uid=buck_version_uid):
use_buckd = False
self._reporter.no_buckd_reason = "daemon_failure"
logging.warning(
"Not using buckd because daemon failed to start."
)
env = self._environ_for_buck()
env["BUCK_BUILD_ID"] = build_id
self._reporter.is_buckd = use_buckd
run_fn = (self._run_with_nailgun
if use_buckd else self._run_without_nailgun)
self._unpack_modules()
exit_code = self._execute_command_and_maybe_run_target(
run_fn, env, argv, java11_test_mode)
# Most shells return process termination with signal as
# 128 + N, where N is the signal. However Python's subprocess
# call returns them as negative numbers. Buck binary protocol
# uses shell's convention, so convert
if exit_code < 0:
exit_code = 128 + (-1 * exit_code)
return exit_code
def launch_buckd(self, buck_version_uid=None):
with Tracing("BuckRepo.launch_buckd"):
self._setup_watchman_watch()
if buck_version_uid is None:
buck_version_uid = self._get_buck_version_uid()
# Override self._tmp_dir to a long lived directory.
buckd_tmp_dir = self._buck_project.create_buckd_tmp_dir()
ngserver_output_path = os.path.join(buckd_tmp_dir, "ngserver-out")
"""
Use SoftRefLRUPolicyMSPerMB for immediate GC of javac output.
Set timeout to 60s (longer than the biggest GC pause seen for a 2GB
heap) and GC target to 15s. This means that the GC has to miss its
target by 100% or many 500ms heartbeats must be missed before a client
disconnection occurs. Specify port 0 to allow Nailgun to find an
available port, then parse the port number out of the first log entry.
"""
command = ["buckd"]
extra_default_options = [
"-Dbuck.buckd_launch_time_nanos={0}".format(monotonic_time_nanos()),
"-XX:MaxGCPauseMillis={0}".format(GC_MAX_PAUSE_TARGET),
"-XX:SoftRefLRUPolicyMSPerMB=0",
# Stop Java waking up every 50ms to collect thread
# statistics; doing it once every five seconds is much
# saner for a long-lived daemon.
"-XX:PerfDataSamplingInterval=5000",
# Do not touch most signals
"-Xrs",
# Likewise, waking up once per second just in case
# there's some rebalancing to be done is silly.
"-XX:+UnlockDiagnosticVMOptions",
"-XX:GuaranteedSafepointInterval=5000",
"-Djava.io.tmpdir={0}".format(buckd_tmp_dir),
"-Dcom.martiansoftware.nailgun.NGServer.outputPath={0}".format(ngserver_output_path),
]
if is_java8_or_9():
extra_default_options.extend(["-XX:+UseG1GC", "-XX:MaxHeapFreeRatio=40"])
command.extend(self._get_java_args(buck_version_uid, extra_default_options))
command.append("com.facebook.buck.cli.bootstrapper.ClassLoaderBootstrapper")
command.append("com.facebook.buck.cli.Main$DaemonBootstrap")
command.append("local:.buckd/sock")
command.append("{0}".format(BUCKD_CLIENT_TIMEOUT_MILLIS))
"""
Change the process group of the child buckd process so that when this
script is interrupted, it does not kill buckd.
"""
def preexec_func():
# Close any open file descriptors to further separate buckd from its
# invoking context (e.g. otherwise we'd hang when running things like
# `ssh localhost buck clean`).
dev_null_fd = os.open("/dev/null", os.O_RDWR)
os.dup2(dev_null_fd, 0)
os.dup2(dev_null_fd, 1)
os.dup2(dev_null_fd, 2)
os.close(dev_null_fd)
buck_socket_path = self._buck_project.get_buckd_socket_path()
# Make sure the Unix domain socket doesn't exist before this call.
try:
os.unlink(buck_socket_path)
except OSError as e:
if e.errno == errno.ENOENT:
# Socket didn't previously exist.
pass
else:
raise e
process = subprocess.Popen(
command,
executable=which("java"),
cwd=self._buck_project.root,
close_fds=True,
preexec_fn=preexec_func,
env=self._environ_for_buck(),
)
self._buck_project.save_buckd_version(buck_version_uid)
self._buck_project.update_buckd_run_count(0)
# Give Java some time to create the listening socket.
for i in range(0, 100):
if not os.path.exists(buck_socket_path):
time.sleep(0.01)
returncode = process.poll()
# If the process hasn't exited yet, everything is working as expected
if returncode is None:
return 0
return returncode
def launch_buck(self, build_id):
with Tracing('BuckRepo.launch_buck'):
if not is_java8_or_9():
WARNING = '\033[93m'
ENDC = '\033[0m'
print(WARNING + "::: Buck requires Java 8 or higher." + ENDC,
file=sys.stderr)
if os_platform == 'darwin':
print("::: Available Java homes:", file=sys.stderr)
check_output(['/usr/libexec/java_home', '-V'])
if not os.environ.get("JAVA_HOME"):
print(WARNING + "::: No Java home selected" + ENDC,
file=sys.stderr)
else:
print(WARNING + "::: Selected Java home:" + ENDC,
file=sys.stderr)
print(WARNING +
"::: {0}".format(os.environ.get("JAVA_HOME")) +
ENDC,
file=sys.stderr)
print(
WARNING +
"::: Select a Java home version 1.8 or higher by setting the JAVA_HOME "
+ "environment variable to point to one" + ENDC,
file=sys.stderr)
print(
WARNING +
"::: Continuing anyway in 30 seconds, but Buck might crash."
+ ENDC,
file=sys.stderr)
time.sleep(30)
if self._command_line.command == "clean" and not self._command_line.is_help(
):
self.kill_buckd()
buck_version_uid = self._get_buck_version_uid()
use_buckd = self._use_buckd()
if not self._command_line.is_help():
has_watchman = bool(which('watchman'))
if use_buckd and has_watchman:
buckd_run_count = self._buck_project.get_buckd_run_count()
running_version = self._buck_project.get_running_buckd_version(
)
new_buckd_run_count = buckd_run_count + 1
if (buckd_run_count == MAX_BUCKD_RUN_COUNT
or running_version != buck_version_uid):
self.kill_buckd()
new_buckd_run_count = 0
if new_buckd_run_count == 0 or not self._is_buckd_running(
):
self.launch_buckd(buck_version_uid=buck_version_uid)
else:
self._buck_project.update_buckd_run_count(
new_buckd_run_count)
elif use_buckd and not has_watchman:
print("Not using buckd because watchman isn't installed.",
file=sys.stderr)
elif not use_buckd:
print("Not using buckd because NO_BUCKD is set.",
file=sys.stderr)
env = self._environ_for_buck()
env['BUCK_BUILD_ID'] = build_id
buck_socket_path = self._buck_project.get_buckd_socket_path()
if use_buckd and self._is_buckd_running() and \
os.path.exists(buck_socket_path):
with Tracing('buck', args={'command': sys.argv[1:]}):
exit_code = 2
last_diagnostic_time = 0
while exit_code == 2:
with NailgunConnection(
'local:.buckd/sock',
cwd=self._buck_project.root) as c:
exit_code = c.send_command(
'com.facebook.buck.cli.Main',
sys.argv[1:],
env=env,
cwd=self._buck_project.root)
if exit_code == 2:
now = time.time()
if now - last_diagnostic_time > DAEMON_BUSY_MESSAGE_SECONDS:
print(
'Daemon is busy, waiting for it to become free...',
file=sys.stderr)
last_diagnostic_time = now
time.sleep(1)
return exit_code
command = ["buck"]
extra_default_options = [
"-Djava.io.tmpdir={0}".format(self._tmp_dir)
]
command.extend(
self._get_java_args(buck_version_uid, extra_default_options))
command.append(
"com.facebook.buck.cli.bootstrapper.ClassLoaderBootstrapper")
command.append("com.facebook.buck.cli.Main")
command.extend(sys.argv[1:])
return subprocess.call(command,
cwd=self._buck_project.root,
env=env,
executable=which("java"))
def launch_buckd(self, buck_version_uid=None):
with Tracing('BuckTool.launch_buckd'):
setup_watchman_watch()
if buck_version_uid is None:
buck_version_uid = self._get_buck_version_uid()
# Override self._tmp_dir to a long lived directory.
buckd_tmp_dir = self._buck_project.create_buckd_tmp_dir()
ngserver_output_path = os.path.join(buckd_tmp_dir, 'ngserver-out')
'''
Use SoftRefLRUPolicyMSPerMB for immediate GC of javac output.
Set timeout to 60s (longer than the biggest GC pause seen for a 2GB
heap) and GC target to 15s. This means that the GC has to miss its
target by 100% or many 500ms heartbeats must be missed before a client
disconnection occurs. Specify port 0 to allow Nailgun to find an
available port, then parse the port number out of the first log entry.
'''
command = ["buckd"]
extra_default_options = [
"-Dbuck.buckd_launch_time_nanos={0}".format(
monotonic_time_nanos()),
"-XX:MaxGCPauseMillis={0}".format(GC_MAX_PAUSE_TARGET),
"-XX:SoftRefLRUPolicyMSPerMB=0",
# Stop Java waking up every 50ms to collect thread
# statistics; doing it once every five seconds is much
# saner for a long-lived daemon.
"-XX:PerfDataSamplingInterval=5000",
# Do not touch most signals
"-Xrs",
# Likewise, waking up once per second just in case
# there's some rebalancing to be done is silly.
"-XX:+UnlockDiagnosticVMOptions",
"-XX:GuaranteedSafepointInterval=5000",
"-Djava.io.tmpdir={0}".format(buckd_tmp_dir),
"-Dcom.martiansoftware.nailgun.NGServer.outputPath={0}".format(
ngserver_output_path),
"-XX:+UseG1GC",
"-XX:MaxHeapFreeRatio=40",
]
command.extend(
self._get_java_args(buck_version_uid, extra_default_options))
command.append(
"com.facebook.buck.cli.bootstrapper.ClassLoaderBootstrapper")
command.append("com.facebook.buck.cli.Main$DaemonBootstrap")
command.append(self._buck_project.get_buckd_transport_address())
command.append("{0}".format(BUCKD_CLIENT_TIMEOUT_MILLIS))
buckd_transport_file_path = self._buck_project.get_buckd_transport_file_path(
)
if os.name == 'nt':
preexec_fn = None
# https://msdn.microsoft.com/en-us/library/windows/desktop/ms684863.aspx#DETACHED_PROCESS
DETACHED_PROCESS = 0x00000008
creationflags = DETACHED_PROCESS
else:
# Make sure the Unix domain socket doesn't exist before this call.
try:
os.unlink(buckd_transport_file_path)
except OSError as e:
if e.errno == errno.ENOENT:
# Socket didn't previously exist.
pass
else:
raise e
'''
Change the process group of the child buckd process so that when this
script is interrupted, it does not kill buckd.
'''
def preexec_fn():
# Close any open file descriptors to further separate buckd from its
# invoking context (e.g. otherwise we'd hang when running things like
# `ssh localhost buck clean`).
dev_null_fd = os.open("/dev/null", os.O_RDWR)
os.dup2(dev_null_fd, 0)
os.dup2(dev_null_fd, 1)
os.dup2(dev_null_fd, 2)
os.close(dev_null_fd)
creationflags = 0
process = subprocess.Popen(command,
executable=which("java"),
cwd=self._buck_project.root,
close_fds=True,
preexec_fn=preexec_fn,
env=self._environ_for_buck(),
creationflags=creationflags)
self._buck_project.save_buckd_version(buck_version_uid)
# Give Java some time to create the listening socket.
for i in range(0, 300):
if not transport_exists(buckd_transport_file_path):
time.sleep(0.01)
returncode = process.poll()
# If the process hasn't exited yet, everything is working as expected
if returncode is None:
return 0
return returncode
def test_without_java_home(self):
self.assertEquals(get_java_path().lower(), which("java").lower())
def launch_buck(self, build_id):
with Tracing('BuckTool.launch_buck'):
with JvmCrashLogger(self, self._buck_project.root):
if self._command_line.command == "clean" and \
not self._command_line.is_help() and \
not self._command_line.is_oop_javac():
self.kill_buckd()
buck_version_uid = self._get_buck_version_uid()
use_buckd = self._use_buckd
if not self._command_line.is_help(
) and not self._command_line.is_oop_javac():
has_watchman = bool(which('watchman'))
if use_buckd and has_watchman:
running_version = self._buck_project.get_running_buckd_version(
)
if running_version != buck_version_uid:
self.kill_buckd()
if not self._is_buckd_running():
self.launch_buckd(
buck_version_uid=buck_version_uid)
elif use_buckd and not has_watchman:
print(
"Not using buckd because watchman isn't installed.",
file=sys.stderr)
elif not use_buckd:
print("Not using buckd because NO_BUCKD is set.",
file=sys.stderr)
env = self._environ_for_buck()
env['BUCK_BUILD_ID'] = build_id
if use_buckd and self._is_buckd_running():
with Tracing('buck', args={'command': sys.argv[1:]}):
exit_code = 2
last_diagnostic_time = 0
while exit_code == 2:
with NailgunConnection(
self._buck_project.
get_buckd_transport_address(),
cwd=self._buck_project.root) as c:
now = int(round(time.time() * 1000))
env['BUCK_PYTHON_SPACE_INIT_TIME'] = \
str(now - self._init_timestamp)
exit_code = c.send_command(
'com.facebook.buck.cli.Main',
sys.argv[1:],
env=env,
cwd=self._buck_project.root)
if exit_code == 2:
env['BUCK_BUILD_ID'] = str(uuid.uuid4())
now = time.time()
if now - last_diagnostic_time > DAEMON_BUSY_MESSAGE_SECONDS:
print(
'Daemon is busy, waiting for it to become free...',
file=sys.stderr)
last_diagnostic_time = now
time.sleep(1)
return exit_code
command = ["buck"]
extra_default_options = [
"-Djava.io.tmpdir={0}".format(self._tmp_dir),
"-XX:SoftRefLRUPolicyMSPerMB=0",
"-XX:+UseG1GC",
]
command.extend(
self._get_java_args(buck_version_uid,
extra_default_options))
command.append(
"com.facebook.buck.cli.bootstrapper.ClassLoaderBootstrapper"
)
if self._command_line.is_oop_javac():
command.append("com.facebook.buck.oop_javac.Main")
else:
command.append("com.facebook.buck.cli.Main")
command.extend(sys.argv[1:])
now = int(round(time.time() * 1000))
env['BUCK_PYTHON_SPACE_INIT_TIME'] = str(now -
self._init_timestamp)
if True:
java = which("java")
if java is None:
raise BuckToolException('Could not find java on $PATH')
with Tracing('buck', args={'command': command}):
buck_exit_code = subprocess.call(
command,
cwd=self._buck_project.root,
env=env,
executable=java)
return buck_exit_code
def launch_buck(self, build_id):
with Tracing('BuckRepo.launch_buck'):
if not is_java8_or_9():
WARNING = '\033[93m'
ENDC = '\033[0m'
print(WARNING + "::: Buck requires Java 8 or higher." + ENDC, file=sys.stderr)
if os_platform == 'darwin':
print("::: Available Java homes:", file=sys.stderr)
check_output(['/usr/libexec/java_home', '-V'])
if not os.environ.get("JAVA_HOME"):
print(WARNING + "::: No Java home selected" + ENDC, file=sys.stderr)
else:
print(WARNING + "::: Selected Java home:" + ENDC, file=sys.stderr)
print(
WARNING + "::: {0}".format(os.environ.get("JAVA_HOME")) + ENDC,
file=sys.stderr)
print(
WARNING +
"::: Select a Java home version 1.8 or higher by setting the JAVA_HOME " +
"environment variable to point to one" + ENDC,
file=sys.stderr)
print(
WARNING + "::: Continuing anyway in 30 seconds, but Buck might crash." + ENDC,
file=sys.stderr)
time.sleep(30)
if self._command_line.command == "clean" and not self._command_line.is_help():
self.kill_buckd()
buck_version_uid = self._get_buck_version_uid()
use_buckd = self._use_buckd()
if not self._command_line.is_help():
has_watchman = bool(which('watchman'))
if use_buckd and has_watchman:
buckd_run_count = self._buck_project.get_buckd_run_count()
running_version = self._buck_project.get_running_buckd_version()
new_buckd_run_count = buckd_run_count + 1
if (buckd_run_count == MAX_BUCKD_RUN_COUNT or
running_version != buck_version_uid):
self.kill_buckd()
new_buckd_run_count = 0
if new_buckd_run_count == 0 or not self._is_buckd_running():
self.launch_buckd(buck_version_uid=buck_version_uid)
else:
self._buck_project.update_buckd_run_count(new_buckd_run_count)
elif use_buckd and not has_watchman:
print("Not using buckd because watchman isn't installed.",
file=sys.stderr)
elif not use_buckd:
print("Not using buckd because NO_BUCKD is set.",
file=sys.stderr)
env = self._environ_for_buck()
env['BUCK_BUILD_ID'] = build_id
buck_socket_path = self._buck_project.get_buckd_socket_path()
if use_buckd and self._is_buckd_running() and \
os.path.exists(buck_socket_path):
with Tracing('buck', args={'command': sys.argv[1:]}):
exit_code = 2
last_diagnostic_time = 0
while exit_code == 2:
with NailgunConnection('local:.buckd/sock',
cwd=self._buck_project.root) as c:
exit_code = c.send_command(
'com.facebook.buck.cli.Main',
sys.argv[1:],
env=env,
cwd=self._buck_project.root)
if exit_code == 2:
now = time.time()
if now - last_diagnostic_time > DAEMON_BUSY_MESSAGE_SECONDS:
print('Daemon is busy, waiting for it to become free...',
file=sys.stderr)
last_diagnostic_time = now
time.sleep(1)
return exit_code
command = ["buck"]
extra_default_options = [
"-Djava.io.tmpdir={0}".format(self._tmp_dir)
]
command.extend(self._get_java_args(buck_version_uid, extra_default_options))
command.append("com.facebook.buck.cli.bootstrapper.ClassLoaderBootstrapper")
command.append("com.facebook.buck.cli.Main")
command.extend(sys.argv[1:])
return subprocess.call(command,
cwd=self._buck_project.root,
env=env,
executable=which("java"))
def launch_buck(self, build_id, java_path, argv):
with Tracing("BuckTool.launch_buck"):
with JvmCrashLogger(self, self._buck_project.root):
self._reporter.build_id = build_id
try:
repository = self._get_repository()
self._reporter.repository = repository
except Exception as e:
# _get_repository() is only for reporting,
# so skip on error
logging.warning("Failed to get repo name: " + str(e))
if (
self._command_line.command == "clean"
and not self._command_line.is_help()
):
self.kill_buckd()
buck_version_uid = self._get_buck_version_uid()
self._reporter.buck_version = buck_version_uid
if self._command_line.is_version():
print("buck version {}".format(buck_version_uid))
return 0
use_buckd = self._use_buckd
if not use_buckd:
logging.warning("Not using buckd because NO_BUCKD is set.")
self._reporter.no_buckd_reason = "explicit"
if use_buckd and self._command_line.is_help():
use_buckd = False
self._reporter.no_buckd_reason = "help"
if use_buckd:
has_watchman = bool(which("watchman"))
if not has_watchman:
use_buckd = False
self._reporter.no_buckd_reason = "watchman"
logging.warning(
"Not using buckd because watchman isn't installed."
)
if use_buckd:
need_start = True
running_version = self._buck_project.get_running_buckd_version()
running_jvm_args = self._buck_project.get_running_buckd_jvm_args()
jvm_args = self._get_java_args(buck_version_uid)
if running_version is None:
logging.info("Starting new Buck daemon...")
elif running_version != buck_version_uid:
logging.info(
"Restarting Buck daemon because Buck version has changed..."
)
elif not self._is_buckd_running():
logging.info(
"Unable to connect to Buck daemon, restarting it..."
)
elif jvm_args != running_jvm_args:
logging.info(
"Restarting Buck daemon because JVM args have changed..."
)
else:
need_start = False
if need_start:
self.kill_buckd()
if not self.launch_buckd(
java_path, jvm_args, buck_version_uid=buck_version_uid
):
use_buckd = False
self._reporter.no_buckd_reason = "daemon_failure"
logging.warning(
"Not using buckd because daemon failed to start."
)
env = self._environ_for_buck()
env["BUCK_BUILD_ID"] = build_id
self._reporter.is_buckd = use_buckd
run_fn = (
self._run_with_nailgun if use_buckd else self._run_without_nailgun
)
self._unpack_modules()
exit_code = self._execute_command_and_maybe_run_target(
run_fn, java_path, env, argv
)
# Most shells return process termination with signal as
# 128 + N, where N is the signal. However Python's subprocess
# call returns them as negative numbers. Buck binary protocol
# uses shell's convention, so convert
if exit_code < 0:
exit_code = 128 + (-1 * exit_code)
return exit_code
def launch_buckd(self, buck_version_uid=None):
with Tracing('BuckTool.launch_buckd'):
setup_watchman_watch()
if buck_version_uid is None:
buck_version_uid = self._get_buck_version_uid()
# Override self._tmp_dir to a long lived directory.
buckd_tmp_dir = self._buck_project.create_buckd_tmp_dir()
ngserver_output_path = os.path.join(buckd_tmp_dir, 'ngserver-out')
"""
Use SoftRefLRUPolicyMSPerMB for immediate GC of javac output.
Set timeout to 60s (longer than the biggest GC pause seen for a 2GB
heap) and GC target to 15s. This means that the GC has to miss its
target by 100% or many 500ms heartbeats must be missed before a client
disconnection occurs. Specify port 0 to allow Nailgun to find an
available port, then parse the port number out of the first log entry.
"""
command = ["buckd"]
extra_default_options = [
"-Dbuck.buckd_launch_time_nanos={0}".format(monotonic_time_nanos()),
"-Dfile.encoding=UTF-8",
"-XX:MaxGCPauseMillis={0}".format(GC_MAX_PAUSE_TARGET),
"-XX:SoftRefLRUPolicyMSPerMB=0",
# Stop Java waking up every 50ms to collect thread
# statistics; doing it once every five seconds is much
# saner for a long-lived daemon.
"-XX:PerfDataSamplingInterval=5000",
# Do not touch most signals
"-Xrs",
# Likewise, waking up once per second just in case
# there's some rebalancing to be done is silly.
"-XX:+UnlockDiagnosticVMOptions",
"-XX:GuaranteedSafepointInterval=5000",
"-Djava.io.tmpdir={0}".format(buckd_tmp_dir),
"-Dcom.martiansoftware.nailgun.NGServer.outputPath={0}".format(
ngserver_output_path),
"-XX:+UseG1GC",
"-XX:MaxHeapFreeRatio=40",
]
command.extend(self._get_java_args(buck_version_uid, extra_default_options))
command.append("com.facebook.buck.cli.bootstrapper.ClassLoaderBootstrapper")
command.append("com.facebook.buck.cli.Main$DaemonBootstrap")
command.append(self._buck_project.get_buckd_transport_address())
command.append("{0}".format(BUCKD_CLIENT_TIMEOUT_MILLIS))
buckd_transport_file_path = self._buck_project.get_buckd_transport_file_path()
if os.name == 'nt':
preexec_fn = None
# https://msdn.microsoft.com/en-us/library/windows/desktop/ms684863.aspx#DETACHED_PROCESS
DETACHED_PROCESS = 0x00000008
creationflags = DETACHED_PROCESS
else:
"""
Change the process group of the child buckd process so that when this
script is interrupted, it does not kill buckd.
"""
def preexec_fn():
# Close any open file descriptors to further separate buckd from its
# invoking context (e.g. otherwise we'd hang when running things like
# `ssh localhost buck clean`).
dev_null_fd = os.open("/dev/null", os.O_RDWR)
os.dup2(dev_null_fd, 0)
os.dup2(dev_null_fd, 1)
os.dup2(dev_null_fd, 2)
os.close(dev_null_fd)
creationflags = 0
process = subprocess.Popen(
command,
executable=which("java"),
cwd=self._buck_project.root,
close_fds=True,
preexec_fn=preexec_fn,
env=self._environ_for_buck(),
creationflags=creationflags)
self._buck_project.save_buckd_version(buck_version_uid)
# Give Java some time to create the listening socket.
wait_seconds = 0.01
repetitions = int(BUCKD_STARTUP_TIMEOUT_MILLIS / 1000.0 / wait_seconds)
for i in range(repetitions):
if transport_exists(buckd_transport_file_path):
break
time.sleep(wait_seconds)
if not transport_exists(buckd_transport_file_path):
return False
returncode = process.poll()
# If the process hasn't exited yet, everything is working as expected
if returncode is None:
return True
return False
def launch_buckd(self, buck_version_uid=None):
with Tracing('BuckTool.launch_buckd'):
setup_watchman_watch()
if buck_version_uid is None:
buck_version_uid = self._get_buck_version_uid()
# Override self._tmp_dir to a long lived directory.
buckd_tmp_dir = self._buck_project.create_buckd_tmp_dir()
ngserver_output_path = os.path.join(buckd_tmp_dir, 'ngserver-out')
"""
Use SoftRefLRUPolicyMSPerMB for immediate GC of javac output.
Set timeout to 60s (longer than the biggest GC pause seen for a 2GB
heap) and GC target to 15s. This means that the GC has to miss its
target by 100% or many 500ms heartbeats must be missed before a client
disconnection occurs. Specify port 0 to allow Nailgun to find an
available port, then parse the port number out of the first log entry.
"""
command = ["buckd"]
extra_default_options = [
"-Dbuck.buckd_launch_time_nanos={0}".format(
monotonic_time_nanos()),
"-Dfile.encoding=UTF-8",
"-XX:MaxGCPauseMillis={0}".format(GC_MAX_PAUSE_TARGET),
"-XX:SoftRefLRUPolicyMSPerMB=0",
# Stop Java waking up every 50ms to collect thread
# statistics; doing it once every five seconds is much
# saner for a long-lived daemon.
"-XX:PerfDataSamplingInterval=5000",
# Do not touch most signals
"-Xrs",
# Likewise, waking up once per second just in case
# there's some rebalancing to be done is silly.
"-XX:+UnlockDiagnosticVMOptions",
"-XX:GuaranteedSafepointInterval=5000",
"-Djava.io.tmpdir={0}".format(buckd_tmp_dir),
"-Dcom.martiansoftware.nailgun.NGServer.outputPath={0}".format(
ngserver_output_path),
"-XX:+UseG1GC",
"-XX:MaxHeapFreeRatio=40",
]
command.extend(
self._get_java_args(buck_version_uid, extra_default_options))
command.append(
"com.facebook.buck.cli.bootstrapper.ClassLoaderBootstrapper")
command.append("com.facebook.buck.cli.Main$DaemonBootstrap")
command.append(self._buck_project.get_buckd_transport_address())
command.append("{0}".format(BUCKD_CLIENT_TIMEOUT_MILLIS))
buckd_transport_file_path = self._buck_project.get_buckd_transport_file_path(
)
if os.name == 'nt':
# https://msdn.microsoft.com/en-us/library/windows/desktop/ms684863.aspx#DETACHED_PROCESS
DETACHED_PROCESS = 0x00000008
creationflags = DETACHED_PROCESS
# do not redirect output for Windows as it deadlocks
stdin = None
stdout = None
stderr = None
close_fds = True
else:
"""
Change the process group of the child buckd process so that when this
script is interrupted, it does not kill buckd.
"""
creationflags = 0
stdin = open(os.devnull, mode='r')
stdout = open(self._buck_project.get_buckd_stdout(),
mode='w+b',
buffering=0)
stderr = open(self._buck_project.get_buckd_stderr(),
mode='w+b',
buffering=0)
close_fds = False
process = subprocess.Popen(
command,
executable=which("java"),
cwd=self._buck_project.root,
env=self._environ_for_buck(),
creationflags=creationflags,
close_fds=close_fds,
stdin=stdin,
stdout=stdout,
stderr=stderr,
)
self._buck_project.save_buckd_version(buck_version_uid)
# Give Java some time to create the listening socket.
wait_seconds = 0.01
repetitions = int(BUCKD_STARTUP_TIMEOUT_MILLIS / 1000.0 /
wait_seconds)
for i in range(repetitions):
if transport_exists(buckd_transport_file_path):
break
time.sleep(wait_seconds)
if not transport_exists(buckd_transport_file_path):
return False
returncode = process.poll()
# If the process hasn't exited yet, everything is working as expected
if returncode is None:
return True
return False
def launch_buck(self, build_id):
with Tracing('BuckRepo.launch_buck'):
if self._command_line.command == "clean" and not self._command_line.is_help(
):
self.kill_buckd()
buck_version_uid = self._get_buck_version_uid()
use_buckd = self._use_buckd()
if not self._command_line.is_help():
has_watchman = bool(which('watchman'))
if use_buckd and has_watchman:
buckd_run_count = self._buck_project.get_buckd_run_count()
running_version = self._buck_project.get_running_buckd_version(
)
new_buckd_run_count = buckd_run_count + 1
if (buckd_run_count == MAX_BUCKD_RUN_COUNT
or running_version != buck_version_uid):
self.kill_buckd()
new_buckd_run_count = 0
if new_buckd_run_count == 0 or not self._is_buckd_running(
):
self.launch_buckd(buck_version_uid=buck_version_uid)
else:
self._buck_project.update_buckd_run_count(
new_buckd_run_count)
elif use_buckd and not has_watchman:
print("Not using buckd because watchman isn't installed.",
file=sys.stderr)
elif not use_buckd:
print("Not using buckd because NO_BUCKD is set.",
file=sys.stderr)
env = self._environ_for_buck()
env['BUCK_BUILD_ID'] = build_id
buck_socket_path = self._buck_project.get_buckd_socket_path()
if use_buckd and self._is_buckd_running() and \
os.path.exists(buck_socket_path):
with Tracing('buck', args={'command': sys.argv[1:]}):
exit_code = 2
last_diagnostic_time = 0
while exit_code == 2:
with NailgunConnection(
'local:.buckd/sock',
cwd=self._buck_project.root) as c:
exit_code = c.send_command(
'com.facebook.buck.cli.Main',
sys.argv[1:],
env=env,
cwd=self._buck_project.root)
if exit_code == 2:
now = time.time()
if now - last_diagnostic_time > DAEMON_BUSY_MESSAGE_SECONDS:
print(
'Daemon is busy, waiting for it to become free...',
file=sys.stderr)
last_diagnostic_time = now
time.sleep(1)
return exit_code
command = ["buck"]
extra_default_options = [
"-Djava.io.tmpdir={0}".format(self._tmp_dir)
]
command.extend(
self._get_java_args(buck_version_uid, extra_default_options))
command.append(
"com.facebook.buck.cli.bootstrapper.ClassLoaderBootstrapper")
command.append("com.facebook.buck.cli.Main")
command.extend(sys.argv[1:])
if True:
buck_exit_code = subprocess.call(command,
cwd=self._buck_project.root,
env=env,
executable=which("java"))
return buck_exit_code
def launch_buck(self, build_id):
with Tracing('BuckTool.launch_buck'):
with JvmCrashLogger(self, self._buck_project.root):
self._reporter.build_id = build_id
try:
repository = self._get_repository()
self._reporter.repository = repository
except Exception as e:
# _get_repository() is only for reporting,
# so skip on error
logging.warning('Failed to get repo name: ' + str(e))
if self._command_line.command == "clean" and \
not self._command_line.is_help():
self.kill_buckd()
buck_version_uid = self._get_buck_version_uid()
self._reporter.buck_version = buck_version_uid
if self._command_line.is_version():
print("buck version {}".format(buck_version_uid))
return 0
use_buckd = self._use_buckd
if not use_buckd:
logging.warning("Not using buckd because NO_BUCKD is set.")
self._reporter.no_buckd_reason = "explicit"
if use_buckd and self._command_line.is_help():
use_buckd = False
self._reporter.no_buckd_reason = "help"
if use_buckd:
has_watchman = bool(which('watchman'))
if not has_watchman:
use_buckd = False
self._reporter.no_buckd_reason = "watchman"
logging.warning("Not using buckd because watchman isn't installed.")
if use_buckd:
running_version = self._buck_project.get_running_buckd_version()
if running_version != buck_version_uid or not self._is_buckd_running():
self.kill_buckd()
if not self.launch_buckd(buck_version_uid=buck_version_uid):
use_buckd = False
self._reporter.no_buckd_reason = "daemon_failure"
logging.warning("Not using buckd because daemon failed to start.")
env = self._environ_for_buck()
env['BUCK_BUILD_ID'] = build_id
self._reporter.is_buckd = use_buckd
run_fn = self._run_with_nailgun if use_buckd else self._run_without_nailgun
self._unpack_modules()
exit_code = self._execute_command_and_maybe_run_target(
run_fn, env)
# Most shells return process termination with signal as
# 128 + N, where N is the signal. However Python's subprocess
# call returns them as negative numbers. Buck binary protocol
# uses shell's convention, so convert
if (exit_code < 0):
exit_code = 128 + (-1 * exit_code)
return exit_code
