def main(argv):
install_signal_handlers()
try:
java_home = os.getenv("JAVA_HOME", "")
path = os.getenv("PATH", "")
if java_home:
pathsep = os.pathsep
os.environ["PATH"] = os.path.join(java_home, 'bin') + pathsep + path
tracing_dir = None
build_id = str(uuid.uuid4())
with Tracing("main"):
with BuckProject.from_current_dir() as project:
tracing_dir = os.path.join(project.get_buck_out_log_dir(), 'traces')
# Try to detect if we're running a PEX by checking if we were invoked
# via a zip file.
if zipfile.is_zipfile(argv[0]):
from buck_package import BuckPackage
buck_repo = BuckPackage(project)
else:
from buck_repo import BuckRepo
buck_repo = BuckRepo(THIS_DIR, project)
# If 'kill' is the second argument, shut down the buckd process.
if sys.argv[1:] == ['kill']:
buck_repo.kill_buckd()
return 0
return buck_repo.launch_buck(build_id)
finally:
if tracing_dir:
Tracing.write_to_dir(tracing_dir, build_id)
def main(argv):
try:
java_home = os.getenv("JAVA_HOME", "")
path = os.getenv("PATH", "")
if java_home:
pathsep = os.pathsep
if sys.platform == 'cygwin':
pathsep = ';'
os.environ["PATH"] = os.path.join(java_home, 'bin') + pathsep + path
tracing_dir = None
build_id = str(uuid.uuid4())
with Tracing("main"):
with BuckProject.from_current_dir() as project:
tracing_dir = os.path.join(project.get_buck_out_log_dir(), 'traces')
# Try to detect if we're running a PEX by checking if we were invoked
# via a zip file.
if zipfile.is_zipfile(argv[0]):
from buck_package import BuckPackage
buck_repo = BuckPackage(project)
else:
from buck_repo import BuckRepo
buck_repo = BuckRepo(THIS_DIR, project)
return buck_repo.launch_buck(build_id)
finally:
if tracing_dir:
Tracing.write_to_dir(tracing_dir, build_id)
def main(argv):
def get_repo(p):
# Try to detect if we're running a PEX by checking if we were invoked
# via a zip file.
if zipfile.is_zipfile(argv[0]):
from buck_package import BuckPackage
return BuckPackage(p)
else:
from buck_repo import BuckRepo
return BuckRepo(THIS_DIR, p)
install_signal_handlers()
try:
tracing_dir = None
build_id = str(uuid.uuid4())
with Tracing("main"):
with BuckProject.from_current_dir() as project:
tracing_dir = os.path.join(project.get_buck_out_log_dir(), 'traces')
with get_repo(project) as buck_repo:
# If 'kill' is the second argument, shut down the buckd process.
if sys.argv[1:] == ['kill']:
buck_repo.kill_buckd()
return 0
return buck_repo.launch_buck(build_id)
finally:
if tracing_dir:
Tracing.write_to_dir(tracing_dir, build_id)
def main(argv, reporter):
# Change environment at startup to ensure we don't have any other threads
# running yet.
# We set BUCK_ROOT_BUILD_ID to ensure that if we're called in a nested fashion
# from, say, a genrule, we do not reuse the UUID, and logs do not end up with
# confusing / incorrect data
# TODO: remove ability to inject BUCK_BUILD_ID completely. It mostly causes
# problems, and is not a generally useful feature for users.
if "BUCK_BUILD_ID" in os.environ and "BUCK_ROOT_BUILD_ID" not in os.environ:
build_id = os.environ["BUCK_BUILD_ID"]
else:
build_id = str(uuid.uuid4())
if "BUCK_ROOT_BUILD_ID" not in os.environ:
os.environ["BUCK_ROOT_BUILD_ID"] = build_id
java_version_status_queue = Queue(maxsize=1)
def get_repo(p):
# Try to detect if we're running a PEX by checking if we were invoked
# via a zip file.
if zipfile.is_zipfile(argv[0]):
from buck_package import BuckPackage
return BuckPackage(p, reporter)
else:
from buck_repo import BuckRepo
return BuckRepo(THIS_DIR, p, reporter)
# If 'killall' is the second argument, shut down all the buckd processes
if argv[1:] == ["killall"]:
return killall_buck(reporter)
install_signal_handlers()
try:
tracing_dir = None
reporter.build_id = build_id
with Tracing("main"):
with BuckProject.from_current_dir() as project:
tracing_dir = os.path.join(project.get_buck_out_log_dir(), "traces")
with get_repo(project) as buck_repo:
required_java_version = buck_repo.get_buck_compiled_java_version()
java_path = get_java_path(required_java_version)
_try_to_verify_java_version_off_thread(
java_version_status_queue, java_path, required_java_version
)
# If 'kill' is the second argument, shut down the buckd
# process
if argv[1:] == ["kill"]:
buck_repo.kill_buckd()
return ExitCode.SUCCESS
return buck_repo.launch_buck(build_id, java_path, argv)
finally:
if tracing_dir:
Tracing.write_to_dir(tracing_dir, build_id)
_emit_java_version_warnings_if_any(java_version_status_queue)
def main(argv, reporter):
java_version_status_queue = Queue(maxsize=1)
_try_to_verify_java_version_off_thread(java_version_status_queue)
def get_repo(p):
# Try to detect if we're running a PEX by checking if we were invoked
# via a zip file.
if zipfile.is_zipfile(argv[0]):
from buck_package import BuckPackage
return BuckPackage(p, reporter)
else:
from buck_repo import BuckRepo
return BuckRepo(THIS_DIR, p, reporter)
# If 'killall' is the second argument, shut down all the buckd processes
if sys.argv[1:] == ["killall"]:
return killall_buck(reporter)
install_signal_handlers()
try:
tracing_dir = None
build_id = os.environ.get("BUCK_BUILD_ID", str(uuid.uuid4()))
reporter.build_id = build_id
with Tracing("main"):
with BuckProject.from_current_dir() as project:
tracing_dir = os.path.join(project.get_buck_out_log_dir(), "traces")
with get_repo(project) as buck_repo:
# If 'kill' is the second argument, shut down the buckd
# process
if sys.argv[1:] == ["kill"]:
buck_repo.kill_buckd()
return ExitCode.SUCCESS
return buck_repo.launch_buck(build_id)
finally:
if tracing_dir:
Tracing.write_to_dir(tracing_dir, build_id)
_emit_java_version_warnings_if_any(java_version_status_queue)
def main(argv, reporter):
java_version_status_queue = Queue(maxsize=1)
_try_to_verify_java_version_off_thread(java_version_status_queue)
def get_repo(p):
# Try to detect if we're running a PEX by checking if we were invoked
# via a zip file.
if zipfile.is_zipfile(argv[0]):
from buck_package import BuckPackage
return BuckPackage(p, reporter)
else:
from buck_repo import BuckRepo
return BuckRepo(THIS_DIR, p, reporter)
# If 'killall' is the second argument, shut down all the buckd processes
if sys.argv[1:] == ['killall']:
return killall_buck(reporter)
install_signal_handlers()
try:
tracing_dir = None
build_id = os.environ.get('BUCK_BUILD_ID', str(uuid.uuid4()))
reporter.build_id = build_id
with Tracing("main"):
with BuckProject.from_current_dir() as project:
tracing_dir = os.path.join(project.get_buck_out_log_dir(),
'traces')
with get_repo(project) as buck_repo:
# If 'kill' is the second argument, shut down the buckd
# process
if sys.argv[1:] == ['kill']:
buck_repo.kill_buckd()
return ExitCode.SUCCESS
return buck_repo.launch_buck(build_id)
finally:
if tracing_dir:
Tracing.write_to_dir(tracing_dir, build_id)
_emit_java_version_warnings_if_any(java_version_status_queue)
def setup_watchman_watch():
with Tracing('BuckTool._setup_watchman_watch'):
if not which('watchman'):
message = textwrap.dedent("""\
Watchman not found, please install when using buckd.
See https://github.com/facebook/watchman for details.""")
if sys.platform == "darwin":
message += "\n(brew install watchman on OS X)"
# Bail if watchman isn't installed as we know java's
# FileSystemWatcher will take too long to process events.
raise BuckToolException(message)
logging.debug("Using watchman.")
def from_current_dir():
with Tracing("BuckProject.from_current_dir"):
current_dir = os.getcwd()
if "--version" in sys.argv or "-V" in sys.argv:
return BuckProject(current_dir)
at_root_dir = False
while not at_root_dir:
if os.path.exists(os.path.join(current_dir, ".buckconfig")):
return BuckProject(current_dir)
parent_dir = os.path.dirname(current_dir)
at_root_dir = current_dir == parent_dir
current_dir = parent_dir
raise NoBuckConfigFoundException()
def _get_extra_java_args(self):
with Tracing("BuckRepo._get_extra_java_args"):
modules_dir = os.path.join(self.buck_dir, "ant-out",
"buck-modules")
module_resources_dir = os.path.join(self.buck_dir, "ant-out",
"buck-modules-resources")
return [
"-Dbuck.git_dirty={0}".format(int(
self._get_buck_repo_dirty())),
"-Dpf4j.pluginsDir={0}".format(modules_dir),
"-Dbuck.mode=repository",
"-Dbuck.module.resources={0}".format(module_resources_dir),
]
def kill_buckd(self):
with Tracing('BuckTool.kill_buckd'):
buckd_transport_file_path = self._buck_project.get_buckd_transport_file_path(
)
if transport_exists(buckd_transport_file_path):
buckd_pid = self._buck_project.get_running_buckd_pid()
logging.debug("Shutting down buck daemon.")
wait_socket_close = False
try:
with NailgunConnection(
self._buck_project.get_buckd_transport_address(),
cwd=self._buck_project.root) as c:
c.send_command('ng-stop')
wait_socket_close = True
except NailgunException as e:
if e.code not in (NailgunException.CONNECT_FAILED,
NailgunException.CONNECTION_BROKEN,
NailgunException.UNEXPECTED_CHUNKTYPE):
raise BuckToolException(
'Unexpected error shutting down nailgun server: ' +
str(e))
# If ng-stop command succeeds, wait for buckd process to terminate and for the
# socket to close. On Unix ng-stop always drops the connection and throws.
if wait_socket_close:
for i in range(0, 300):
if not transport_exists(buckd_transport_file_path):
break
time.sleep(0.01)
elif buckd_pid is not None and os.name == 'posix':
# otherwise just wait for up to 5 secs for the process to die
# TODO(buck_team) implement wait for process and hard kill for Windows too
if not wait_for_process_posix(buckd_pid, 5000):
# There is a possibility that daemon is dead for some time but pid file
# still exists and another process is assigned to the same pid. Ideally we
# should check first which process we are killing but so far let's pretend
# this will never happen and kill it with fire anyways.
try:
force_kill_process_posix(buckd_pid)
except Exception as e:
# In the worst case it keeps running multiple daemons simultaneously
# consuming memory. Not the good place to be, but let's just issue a
# warning for now.
logging.warning(
'Error killing running Buck daemon ' + str(e))
if transport_exists(buckd_transport_file_path):
force_close_transport(buckd_transport_file_path)
self._buck_project.clean_up_buckd()
def _execute_command_and_maybe_run_target(self, run_fn, env, argv,
java11_test_mode):
"""
Run a buck command using the specified `run_fn`. If the command is "run", get the path,
args, etc. from the daemon, and raise an exception that tells __main__ to run that binary
"""
with Tracing("buck", args={"command": sys.argv[1:]}):
if os.name == "nt":
# Windows now support the VT100 sequences that are used by the Superconsole, but our
# process must opt into this behavior to enable it.
import ctypes
kernel32 = ctypes.windll.kernel32
STD_OUTPUT_HANDLE = -11
ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004
# These calls can fail because we're not output to a terminal or we're not on a new
# enough version of Windows. That's fine because we fail silently, and the
# Superconsole wasn't going to work anyways.
handle = kernel32.GetStdHandle(STD_OUTPUT_HANDLE)
console_mode = ctypes.c_uint(0)
if kernel32.GetConsoleMode(handle, ctypes.byref(console_mode)):
kernel32.SetConsoleMode(
handle, console_mode.value
| ENABLE_VIRTUAL_TERMINAL_PROCESSING)
argv = argv[1:]
if len(argv) == 0 or argv[0] != "run":
return run_fn(argv, env, java11_test_mode)
else:
with tempfile.NamedTemporaryFile(
dir=self._tmp_dir) as argsfile:
# Splice in location of command file to run outside buckd
argv = [argv[0]] + ["--command-args-file", argsfile.name
] + argv[1:]
exit_code = run_fn(argv, env, java11_test_mode)
if exit_code != 0 or os.path.getsize(argsfile.name) == 0:
# Build failed, so there's nothing to run. Exit normally.
return exit_code
cmd = json.load(argsfile)
path = cmd["path"].encode("utf8")
argv = [arg.encode("utf8") for arg in cmd["argv"]]
envp = {
k.encode("utf8"): v.encode("utf8")
for k, v in cmd["envp"].iteritems()
}
cwd = cmd["cwd"].encode("utf8")
raise ExecuteTarget(path, argv, envp, cwd)
def main(argv, reporter):
java_version = _get_java_version()
if java_version and java_version != REQUIRED_JAVA_VERSION:
_warn_about_wrong_java_version(REQUIRED_JAVA_VERSION, java_version)
def get_repo(p):
# Try to detect if we're running a PEX by checking if we were invoked
# via a zip file.
if zipfile.is_zipfile(argv[0]):
from buck_package import BuckPackage
return BuckPackage(p, reporter)
else:
from buck_repo import BuckRepo
return BuckRepo(THIS_DIR, p, reporter)
# If 'killall' is the second argument, shut down all the buckd processes
if sys.argv[1:] == ['killall']:
return killall_buck(reporter)
install_signal_handlers()
try:
tracing_dir = None
build_id = str(uuid.uuid4())
reporter.build_id = build_id
with Tracing("main"):
with BuckProject.from_current_dir() as project:
tracing_dir = os.path.join(project.get_buck_out_log_dir(),
'traces')
with get_repo(project) as buck_repo:
# If 'kill' is the second argument, shut down the buckd
# process
if sys.argv[1:] == ['kill']:
buck_repo.kill_buckd()
return 0
return buck_repo.launch_buck(build_id)
finally:
if tracing_dir:
Tracing.write_to_dir(tracing_dir, build_id)
def main(argv, reporter):
java_version = _get_java_version()
if java_version and java_version != REQUIRED_JAVA_VERSION:
_warn_about_wrong_java_version(REQUIRED_JAVA_VERSION, java_version)
def get_repo(p):
# Try to detect if we're running a PEX by checking if we were invoked
# via a zip file.
if zipfile.is_zipfile(argv[0]):
from buck_package import BuckPackage
return BuckPackage(p, reporter)
else:
from buck_repo import BuckRepo
return BuckRepo(THIS_DIR, p, reporter)
# If 'killall' is the second argument, shut down all the buckd processes
if sys.argv[1:] == ['killall']:
return killall_buck(reporter)
install_signal_handlers()
try:
tracing_dir = None
build_id = str(uuid.uuid4())
reporter.build_id = build_id
with Tracing("main"):
with BuckProject.from_current_dir() as project:
tracing_dir = os.path.join(project.get_buck_out_log_dir(),
'traces')
with get_repo(project) as buck_repo:
# If 'kill' is the second argument, shut down the buckd
# process
if sys.argv[1:] == ['kill']:
buck_repo.kill_buckd()
return 0
return buck_repo.launch_buck(build_id)
finally:
if tracing_dir:
Tracing.write_to_dir(tracing_dir, build_id)
def _get_java_args(self, version_uid, extra_default_options=[]):
with Tracing('BuckTool._get_java_args'):
java_args = [
"-Xmx{0}m".format(JAVA_MAX_HEAP_SIZE_MB),
"-Djava.awt.headless=true",
"-Djava.util.logging.config.class=com.facebook.buck.cli.bootstrapper.LogConfig",
"-Dbuck.test_util_no_tests_dir=true",
"-Dbuck.version_uid={0}".format(version_uid),
"-Dbuck.buckd_dir={0}".format(self._buck_project.buckd_dir),
"-Dorg.eclipse.jetty.util.log.class=org.eclipse.jetty.util.log.JavaUtilLog",
]
resource_lock_path = self._get_resource_lock_path()
if resource_lock_path is not None:
java_args.append(
"-Dbuck.resource_lock_path={0}".format(resource_lock_path))
for resource in self._get_exported_resources():
if self._has_resource(resource):
java_args.append("-Dbuck.{0}={1}".format(
resource.name, self._get_resource(resource)))
if sys.platform == "darwin":
java_args.append("-Dbuck.enable_objc=true")
java_args.append("-Djava.library.path=" + os.path.dirname(
self._get_resource(Resource("libjcocoa.dylib"))))
if "BUCK_DEBUG_MODE" in os.environ and os.environ.get(
"BUCK_DEBUG_MODE") != "0":
java_args.append("-agentlib:jdwp=transport=dt_socket,"
"server=y,suspend=y,address=8888")
if os.environ.get("BUCK_DEBUG_SOY"):
java_args.append("-Dbuck.soy.debug=true")
java_args.extend(extra_default_options)
if self._buck_project.buck_javaargs:
java_args.extend(shlex.split(self._buck_project.buck_javaargs))
if self._buck_project.buck_javaargs_local:
java_args.extend(
shlex.split(self._buck_project.buck_javaargs_local))
java_args.extend(self._get_extra_java_args())
extra_java_args = os.environ.get("BUCK_EXTRA_JAVA_ARGS")
if extra_java_args:
java_args.extend(shlex.split(extra_java_args))
return java_args
def _get_buck_version_uid(self):
with Tracing('BuckRepo._get_buck_version_uid'):
if self._fake_buck_version:
return self._fake_buck_version
# First try to get the "clean" buck version. If it succeeds,
# return it.
clean_version = buck_version.get_clean_buck_version(
self.buck_dir,
allow_dirty=self._is_buck_repo_dirty_override == "1")
if clean_version is not None:
return clean_version
return buck_version.get_dirty_buck_version(self.buck_dir)
def main(argv):
try:
java_home = os.getenv("JAVA_HOME", "")
path = os.getenv("PATH", "")
if java_home:
pathsep = os.pathsep
if sys.platform == 'cygwin':
pathsep = ';'
os.environ["PATH"] = os.path.join(java_home,
'bin') + pathsep + path
tracing_dir = None
build_id = str(uuid.uuid4())
with Tracing("main"):
with BuckProject.from_current_dir() as project:
tracing_dir = os.path.join(project.get_buck_out_log_dir(),
'traces')
# Try to detect if we're running a PEX by checking if we were invoked
# via a zip file.
if zipfile.is_zipfile(argv[0]):
from buck_package import BuckPackage
buck_repo = BuckPackage(project)
else:
from buck_repo import BuckRepo
buck_repo = BuckRepo(THIS_DIR, project)
# If 'kill' is the second argument, shut down the buckd process.
if sys.argv[1:] == ['kill']:
buck_repo.kill_buckd()
return 0
return buck_repo.launch_buck(build_id)
finally:
if tracing_dir:
Tracing.write_to_dir(tracing_dir, build_id)
def execute(self):
should_run_res_task = self._invoker.check_res_task()
if not should_run_res_task:
self.debug('no need to execute')
return
self.debug('start to execute aapt command...')
self._invoker.fill_dependant_jars()
self._invoker.check_ids_change()
with Tracing("generate_id_keeper_files"):
self._invoker.generate_r_file()
# self._invoker.backup_res_files()
with Tracing("incremental_databinding_process"):
self._invoker.process_databinding(self._original_changed_files, self._changed_files_ref)
with Tracing("run_incremental_aapt_task"):
self._invoker.run_aapt_task()
with Tracing("check_other_modules_resources"):
self._invoker.check_other_modules_resources()
self._invoker.recover_original_file_path()
def _checkout_and_clean(self, revision, branch):
with Tracing('BuckRepo._checkout_and_clean'):
if not self._revision_exists(revision):
print(textwrap.dedent("""\
Required revision {0} is not
available in the local repository.
Buck is fetching updates from git. You can disable this by creating
a '.nobuckcheck' file in your repository, but this might lead to
strange bugs or build failures.""".format(revision)),
file=sys.stderr)
git_command = ['git', 'fetch']
git_command.extend(
['--all'] if not branch else ['origin', branch])
try:
subprocess.check_call(git_command,
stdout=sys.stderr,
cwd=self._buck_dir)
except subprocess.CalledProcessError:
raise BuckToolException(
textwrap.dedent("""\
Failed to fetch Buck updates from git."""))
current_revision = self._get_git_revision()
if current_revision != revision:
print(textwrap.dedent("""\
Buck is at {0}, but should be {1}.
Buck is updating itself. To disable this, add a '.nobuckcheck'
file to your project root. In general, you should only disable
this if you are developing Buck.""".format(
current_revision, revision)),
file=sys.stderr)
try:
subprocess.check_call(
['git', 'checkout', '--quiet', revision],
cwd=self._buck_dir)
except subprocess.CalledProcessError:
raise BuckToolException(
textwrap.dedent("""\
Failed to update Buck to revision {0}.""".format(
revision)))
if os.path.exists(self._build_success_file):
os.remove(self._build_success_file)
ant = self._check_for_ant()
self._run_ant_clean(ant)
raise RestartBuck()
def kill_buckd(self):
with Tracing('BuckTool.kill_buckd'):
buckd_socket_path = self._buck_project.get_buckd_socket_path()
if os.path.exists(buckd_socket_path):
print("Shutting down nailgun server...", file=sys.stderr)
try:
with NailgunConnection('local:.buckd/sock', cwd=self._buck_project.root) as c:
c.send_command('ng-stop')
except NailgunException as e:
if e.code not in (NailgunException.CONNECT_FAILED,
NailgunException.CONNECTION_BROKEN,
NailgunException.UNEXPECTED_CHUNKTYPE):
raise BuckToolException(
'Unexpected error shutting down nailgun server: ' +
str(e))
self._buck_project.clean_up_buckd()
def kill_buckd(self):
with Tracing('BuckTool.kill_buckd'):
buckd_transport_file_path = self._buck_project.get_buckd_transport_file_path()
if transport_exists(buckd_transport_file_path):
logging.debug("Shutting down buck daemon.")
try:
with NailgunConnection(self._buck_project.get_buckd_transport_address(),
cwd=self._buck_project.root) as c:
c.send_command('ng-stop')
except NailgunException as e:
if e.code not in (NailgunException.CONNECT_FAILED,
NailgunException.CONNECTION_BROKEN,
NailgunException.UNEXPECTED_CHUNKTYPE):
raise BuckToolException(
'Unexpected error shutting down nailgun server: ' +
str(e))
self._buck_project.clean_up_buckd()
def tracing(self, numDays, barcode=None):
if not barcode:
return self.index(error="No member selected")
self.checkPermissions()
with self.dbConnect() as dbConnection:
dictVisits = Tracing().getDictVisits(dbConnection, barcode, numDays)
(_, displayName) = self.engine.members.getName(
dbConnection, barcode)
if not displayName:
(_, displayName) = self.engine.guests.getName(
dbConnection, barcode)
return self.template('tracing.mako',
displayName=displayName,
dictVisits=dictVisits,
error="")
def _get_buck_version_uid(self):
with Tracing('BuckRepo._get_buck_version_uid'):
# First try to get the "clean" buck version. If it succeeds,
# return it.
clean_version = buck_version.get_clean_buck_version(
self._buck_dir,
allow_dirty=self._is_buck_repo_dirty_override == "1")
if clean_version is not None:
return clean_version
# Otherwise, if there is a .nobuckcheck file, or if there isn't
# a .buckversion file, fall back to a "dirty" version.
if (self._buck_project.has_no_buck_check
or not self._buck_project.buck_version):
return buck_version.get_dirty_buck_version(self._buck_dir)
if self._has_local_changes():
print(textwrap.dedent("""\
::: Your buck directory has local modifications, and therefore
::: builds will not be able to use a distributed cache.
::: The following files must be either reverted or committed:"""
),
file=sys.stderr)
subprocess.call(['git', 'ls-files', '-m'],
stdout=sys.stderr,
cwd=self._buck_dir)
elif os.environ.get('BUCK_CLEAN_REPO_IF_DIRTY') != 'NO':
print(textwrap.dedent("""\
::: Your local buck directory is dirty, and therefore builds will
::: not be able to use a distributed cache."""),
file=sys.stderr)
if sys.stdout.isatty():
print(
"::: Do you want to clean your buck directory? [y/N]",
file=sys.stderr)
choice = raw_input().lower()
if choice == "y":
subprocess.call(['git', 'clean', '-fd'],
stdout=sys.stderr,
cwd=self._buck_dir)
raise RestartBuck()
return buck_version.get_dirty_buck_version(self._buck_dir)
def _is_buckd_running(self):
with Tracing('BuckRepo._is_buckd_running'):
buckd_socket_path = self._buck_project.get_buckd_socket_path()
if not os.path.exists(buckd_socket_path):
return False
try:
with NailgunConnection('local:.buckd/sock',
stdin=None,
stdout=None,
stderr=None) as c:
c.send_command('ng-stats')
except NailgunException as e:
if e.code == NailgunException.CONNECT_FAILED:
return False
else:
raise
return True
def launch_buck(self, build_id):
with Tracing('BuckTool.launch_buck'):
with JvmCrashLogger(self, self._buck_project.root):
if self._command_line.command == "clean" and \
not self._command_line.is_help():
self.kill_buckd()
buck_version_uid = self._get_buck_version_uid()
if self._command_line.is_version():
print("buck version {}".format(buck_version_uid))
return 0
use_buckd = self._use_buckd
if not self._command_line.is_help():
has_watchman = bool(which('watchman'))
if use_buckd and has_watchman:
running_version = self._buck_project.get_running_buckd_version(
)
if running_version != buck_version_uid:
self.kill_buckd()
if not self._is_buckd_running():
self.launch_buckd(
buck_version_uid=buck_version_uid)
elif use_buckd and not has_watchman:
logging.warning(
"Not using buckd because watchman isn't installed."
)
elif not use_buckd:
logging.warning(
"Not using buckd because NO_BUCKD is set.")
env = self._environ_for_buck()
env['BUCK_BUILD_ID'] = build_id
use_nailgun = use_buckd and self._is_buckd_running()
run_fn = self._run_with_nailgun if use_nailgun else self._run_without_nailgun
self._unpack_modules()
return self._execute_command_and_maybe_run_target(run_fn, env)
def _setup_watchman_watch(self):
with Tracing('BuckRepo._setup_watchman_watch'):
if not which('watchman'):
message = textwrap.dedent("""\
Watchman not found, please install when using buckd.
See https://github.com/facebook/watchman for details.""")
if sys.platform == "darwin":
message += "\n(brew install watchman on OS X)"
# Bail if watchman isn't installed as we know java's
# FileSystemWatcher will take too long to process events.
raise BuckToolException(message)
print("Using watchman.", file=sys.stderr)
try:
check_output(['watchman', 'watch', self._buck_project.root],
stderr=subprocess.STDOUT)
except CalledProcessError as e:
print(e.output, end='', file=sys.stderr)
raise
def _is_buckd_running(self):
with Tracing('BuckTool._is_buckd_running'):
transport_file_path = self._buck_project.get_buckd_transport_file_path()
if not transport_exists(transport_file_path):
return False
try:
with NailgunConnection(
self._buck_project.get_buckd_transport_address(),
stdin=None,
stdout=None,
stderr=None,
cwd=self._buck_project.root) as c:
c.send_command('ng-stats')
except NailgunException as e:
if e.code == NailgunException.CONNECT_FAILED:
return False
else:
raise
return True
def _get_buck_version_uid(self):
with Tracing('BuckRepo._get_buck_version_uid'):
if not self._is_git:
return 'N/A'
if not self._is_dirty():
return self._get_git_revision()
if (self._buck_project.has_no_buck_check
or not self._buck_project.buck_version):
return self._compute_local_hash()
if self._has_local_changes():
print(textwrap.dedent("""\
::: Your buck directory has local modifications, and therefore
::: builds will not be able to use a distributed cache.
::: The following files must be either reverted or committed:"""
),
file=sys.stderr)
subprocess.call(['git', 'ls-files', '-m'],
stdout=sys.stderr,
cwd=self._buck_dir)
elif os.environ.get('BUCK_CLEAN_REPO_IF_DIRTY') != 'NO':
print(textwrap.dedent("""\
::: Your local buck directory is dirty, and therefore builds will
::: not be able to use a distributed cache."""),
file=sys.stderr)
if sys.stdout.isatty():
print(
"::: Do you want to clean your buck directory? [y/N]",
file=sys.stderr)
choice = raw_input().lower()
if choice == "y":
subprocess.call(['git', 'clean', '-fd'],
stdout=sys.stderr,
cwd=self._buck_dir)
raise RestartBuck()
return self._compute_local_hash()
def _run_without_nailgun(self, argv, env):
"""
Run the command by directly invoking `java` (rather than by sending a command via nailgun)
"""
command = ["buck"]
extra_default_options = [
"-Djava.io.tmpdir={0}".format(self._tmp_dir),
"-Dfile.encoding=UTF-8",
"-XX:SoftRefLRUPolicyMSPerMB=0",
"-XX:+UseG1GC",
]
command.extend(self._get_java_args(self._get_buck_version_uid(), extra_default_options))
command.append("com.facebook.buck.cli.bootstrapper.ClassLoaderBootstrapper")
command.append("com.facebook.buck.cli.Main")
command.extend(self._add_args_from_env(argv))
now = int(round(time.time() * 1000))
env['BUCK_PYTHON_SPACE_INIT_TIME'] = str(now - self._init_timestamp)
java = get_java_path()
with Tracing('buck', args={'command': command}):
return subprocess.call(command,
cwd=self._buck_project.root,
env=env,
executable=java)
def _execute_command_and_maybe_run_target(self, run_fn, env):
"""
Run a buck command using the specified `run_fn`. If the command is "run", get the path,
args, etc. from the daemon, and raise an exception that tells __main__ to run that binary
"""
with Tracing('buck', args={'command': sys.argv[1:]}):
argv = sys.argv[1:]
if len(argv) == 0 or argv[0] != 'run':
return run_fn(argv, env)
else:
with tempfile.NamedTemporaryFile(dir=self._tmp_dir) as argsfile:
# Splice in location of command file to run outside buckd
argv = [argv[0]] + ['--command-args-file', argsfile.name] + argv[1:]
exit_code = run_fn(argv, env)
if exit_code != 0 or os.path.getsize(argsfile.name) == 0:
# Build failed, so there's nothing to run. Exit normally.
return exit_code
cmd = json.load(argsfile)
path = cmd['path'].encode('utf8')
argv = [arg.encode('utf8') for arg in cmd['argv']]
envp = {k.encode('utf8'): v.encode('utf8') for k, v in cmd['envp'].iteritems()}
cwd = cmd['cwd'].encode('utf8')
raise ExecuteTarget(path, argv, envp, cwd)
def _is_buckd_running(self):
with Tracing('BuckRepo._is_buckd_running'):
buckd_port = self._buck_project.get_buckd_port()
if buckd_port is None or not buckd_port.isdigit():
return False
buck_client_file = self._get_resource(CLIENT)
command = [buck_client_file]
command.append('ng-stats')
command.append('--nailgun-port')
command.append(buckd_port)
try:
check_output(command,
cwd=self._buck_project.root,
stderr=subprocess.STDOUT)
except CalledProcessError as e:
if e.returncode == NAILGUN_CONNECTION_REFUSED_CODE:
return False
else:
print(e.output, end='', file=sys.stderr)
raise
return True
def _is_buckd_running(self):
with Tracing('BuckRepo._is_buckd_running'):
buckd_socket_path = self._buck_project.get_buckd_socket_path()
if not os.path.exists(buckd_socket_path):
return False
buck_client_file = self._get_resource(CLIENT)
command = [buck_client_file]
command.append('ng-stats')
command.append('--nailgun-server')
command.append('local:.buckd/sock')
try:
check_output(command,
cwd=self._buck_project.root,
stderr=subprocess.STDOUT)
except CalledProcessError as e:
if e.returncode == NAILGUN_CONNECTION_REFUSED_CODE:
return False
else:
print(e.output, end='', file=sys.stderr)
raise
return True
def kill_buckd(self):
with Tracing('BuckRepo.kill_buckd'):
buckd_socket_path = self._buck_project.get_buckd_socket_path()
if os.path.exists(buckd_socket_path):
print("Shutting down nailgun server...", file=sys.stderr)
buck_client_file = self._get_resource(CLIENT)
command = [buck_client_file]
command.append('ng-stop')
command.append('--nailgun-server')
command.append("local:.buckd/sock")
try:
check_output(
command,
cwd=self._buck_project.root,
stderr=subprocess.STDOUT)
except CalledProcessError as e:
if (e.returncode not in
[NAILGUN_CONNECTION_REFUSED_CODE,
NAILGUN_CONNECTION_BROKEN_CODE,
NAILGUN_UNEXPECTED_CHUNK_TYPE]):
print(e.output, end='', file=sys.stderr)
raise
self._buck_project.clean_up_buckd()
def launch_buck(self, build_id):
with Tracing('BuckRepo.launch_buck'):
self.kill_autobuild()
if 'clean' in sys.argv:
self.kill_buckd()
buck_version_uid = self._get_buck_version_uid()
use_buckd = self._use_buckd()
has_watchman = bool(which('watchman'))
if use_buckd and has_watchman:
buckd_run_count = self._buck_project.get_buckd_run_count()
running_version = self._buck_project.get_running_buckd_version(
)
new_buckd_run_count = buckd_run_count + 1
if (buckd_run_count == MAX_BUCKD_RUN_COUNT
or running_version != buck_version_uid):
self.kill_buckd()
new_buckd_run_count = 0
if new_buckd_run_count == 0 or not self._is_buckd_running():
self.launch_buckd(buck_version_uid=buck_version_uid)
else:
self._buck_project.update_buckd_run_count(
new_buckd_run_count)
elif use_buckd and not has_watchman:
print("Not using buckd because watchman isn't installed.",
file=sys.stderr)
env = self._environ_for_buck()
env['BUCK_BUILD_ID'] = build_id
buck_client_file = self._get_resource(CLIENT)
if use_buckd and self._is_buckd_running() and os.path.exists(
buck_client_file):
print("Using buckd.", file=sys.stderr)
buckd_port = self._buck_project.get_buckd_port()
if not buckd_port or not buckd_port.isdigit():
print(
"Daemon port file is corrupt, starting new buck process.",
file=sys.stderr)
self.kill_buckd()
else:
command = [buck_client_file]
command.append("--nailgun-port")
command.append(buckd_port)
command.append("com.facebook.buck.cli.Main")
command.extend(sys.argv[1:])
with Tracing('buck', args={'command': command}):
exit_code = subprocess.call(
command, cwd=self._buck_project.root, env=env)
if exit_code == 2:
print('Daemon is busy, please wait',
'or run "buck kill" to terminate it.',
file=sys.stderr)
return exit_code
command = ["buck"]
command.extend(self._get_java_args(buck_version_uid))
command.append("-Djava.io.tmpdir={0}".format(self._tmp_dir))
command.append(
"com.facebook.buck.cli.bootstrapper.ClassLoaderBootstrapper")
command.append("com.facebook.buck.cli.Main")
command.extend(sys.argv[1:])
return subprocess.call(command,
cwd=self._buck_project.root,
env=env,
executable=which("java"))
def launch_buckd(self, buck_version_uid=None):
with Tracing('BuckRepo.launch_buckd'):
self._setup_watchman_watch()
if buck_version_uid is None:
buck_version_uid = self._get_buck_version_uid()
# Override self._tmp_dir to a long lived directory.
buckd_tmp_dir = self._buck_project.create_buckd_tmp_dir()
ngserver_output_path = os.path.join(buckd_tmp_dir, 'ngserver-out')
'''
Use SoftRefLRUPolicyMSPerMB for immediate GC of javac output.
Set timeout to 60s (longer than the biggest GC pause seen for a 2GB
heap) and GC target to 15s. This means that the GC has to miss its
target by 100% or many 500ms heartbeats must be missed before a client
disconnection occurs. Specify port 0 to allow Nailgun to find an
available port, then parse the port number out of the first log entry.
'''
command = ["buckd"]
command.extend(self._get_java_args(buck_version_uid))
command.append("-Dbuck.buckd_launch_time_nanos={0}".format(
monotonic_time_nanos()))
command.append(
"-XX:MaxGCPauseMillis={0}".format(GC_MAX_PAUSE_TARGET))
command.append("-XX:SoftRefLRUPolicyMSPerMB=0")
command.append("-Djava.io.tmpdir={0}".format(buckd_tmp_dir))
command.append(
"-Dcom.martiansoftware.nailgun.NGServer.outputPath={0}".format(
ngserver_output_path))
command.append(
"com.facebook.buck.cli.bootstrapper.ClassLoaderBootstrapper")
command.append("com.martiansoftware.nailgun.NGServer")
command.append("localhost:0")
command.append("{0}".format(BUCKD_CLIENT_TIMEOUT_MILLIS))
'''
Change the process group of the child buckd process so that when this
script is interrupted, it does not kill buckd.
'''
def preexec_func():
# Close any open file descriptors to further separate buckd from its
# invoking context (e.g. otherwise we'd hang when running things like
# `ssh localhost buck clean`).
# N.B. preexec_func is POSIX-only, and any reasonable
# POSIX system has a /dev/null
os.setpgrp()
dev_null_fd = os.open("/dev/null", os.O_RDWR)
os.dup2(dev_null_fd, 0)
os.dup2(dev_null_fd, 1)
os.dup2(dev_null_fd, 2)
os.close(dev_null_fd)
process = subprocess.Popen(command,
executable=which("java"),
cwd=self._buck_project.root,
close_fds=True,
preexec_fn=preexec_func,
env=self._environ_for_buck())
buckd_port = None
for i in range(100):
if buckd_port:
break
try:
with open(ngserver_output_path) as f:
for line in f:
match = BUCKD_LOG_FILE_PATTERN.match(line)
if match:
buckd_port = match.group(1)
break
except IOError as e:
pass
finally:
time.sleep(0.1)
else:
print(
"nailgun server did not respond after 10s. Aborting buckd.",
file=sys.stderr)
return
self._buck_project.save_buckd_port(buckd_port)
self._buck_project.save_buckd_version(buck_version_uid)
self._buck_project.update_buckd_run_count(0)
def write_contents_to_file(path, contents):
with Tracing("BuckProject.write_contents_to_file", args={"path": path}):
with open(path, "w") as output_file:
output_file.write(str(contents))
def clean_up_buckd(self):
with Tracing("BuckProject.clean_up_buckd"):
if os.path.exists(self.buckd_dir):
file_locks.rmtree_if_can_lock(self.buckd_dir)
