def main(argv): install_signal_handlers() try: java_home = os.getenv("JAVA_HOME", "") path = os.getenv("PATH", "") if java_home: pathsep = os.pathsep os.environ["PATH"] = os.path.join(java_home, 'bin') + pathsep + path tracing_dir = None build_id = str(uuid.uuid4()) with Tracing("main"): with BuckProject.from_current_dir() as project: tracing_dir = os.path.join(project.get_buck_out_log_dir(), 'traces') # Try to detect if we're running a PEX by checking if we were invoked # via a zip file. if zipfile.is_zipfile(argv[0]): from buck_package import BuckPackage buck_repo = BuckPackage(project) else: from buck_repo import BuckRepo buck_repo = BuckRepo(THIS_DIR, project) # If 'kill' is the second argument, shut down the buckd process. if sys.argv[1:] == ['kill']: buck_repo.kill_buckd() return 0 return buck_repo.launch_buck(build_id) finally: if tracing_dir: Tracing.write_to_dir(tracing_dir, build_id)
def main(argv): try: java_home = os.getenv("JAVA_HOME", "") path = os.getenv("PATH", "") if java_home: pathsep = os.pathsep if sys.platform == 'cygwin': pathsep = ';' os.environ["PATH"] = os.path.join(java_home, 'bin') + pathsep + path tracing_dir = None build_id = str(uuid.uuid4()) with Tracing("main"): with BuckProject.from_current_dir() as project: tracing_dir = os.path.join(project.get_buck_out_log_dir(), 'traces') # Try to detect if we're running a PEX by checking if we were invoked # via a zip file. if zipfile.is_zipfile(argv[0]): from buck_package import BuckPackage buck_repo = BuckPackage(project) else: from buck_repo import BuckRepo buck_repo = BuckRepo(THIS_DIR, project) return buck_repo.launch_buck(build_id) finally: if tracing_dir: Tracing.write_to_dir(tracing_dir, build_id)
def main(argv): def get_repo(p): # Try to detect if we're running a PEX by checking if we were invoked # via a zip file. if zipfile.is_zipfile(argv[0]): from buck_package import BuckPackage return BuckPackage(p) else: from buck_repo import BuckRepo return BuckRepo(THIS_DIR, p) install_signal_handlers() try: tracing_dir = None build_id = str(uuid.uuid4()) with Tracing("main"): with BuckProject.from_current_dir() as project: tracing_dir = os.path.join(project.get_buck_out_log_dir(), 'traces') with get_repo(project) as buck_repo: # If 'kill' is the second argument, shut down the buckd process. if sys.argv[1:] == ['kill']: buck_repo.kill_buckd() return 0 return buck_repo.launch_buck(build_id) finally: if tracing_dir: Tracing.write_to_dir(tracing_dir, build_id)
def main(argv, reporter): # Change environment at startup to ensure we don't have any other threads # running yet. # We set BUCK_ROOT_BUILD_ID to ensure that if we're called in a nested fashion # from, say, a genrule, we do not reuse the UUID, and logs do not end up with # confusing / incorrect data # TODO: remove ability to inject BUCK_BUILD_ID completely. It mostly causes # problems, and is not a generally useful feature for users. if "BUCK_BUILD_ID" in os.environ and "BUCK_ROOT_BUILD_ID" not in os.environ: build_id = os.environ["BUCK_BUILD_ID"] else: build_id = str(uuid.uuid4()) if "BUCK_ROOT_BUILD_ID" not in os.environ: os.environ["BUCK_ROOT_BUILD_ID"] = build_id java_version_status_queue = Queue(maxsize=1) def get_repo(p): # Try to detect if we're running a PEX by checking if we were invoked # via a zip file. if zipfile.is_zipfile(argv[0]): from buck_package import BuckPackage return BuckPackage(p, reporter) else: from buck_repo import BuckRepo return BuckRepo(THIS_DIR, p, reporter) # If 'killall' is the second argument, shut down all the buckd processes if argv[1:] == ["killall"]: return killall_buck(reporter) install_signal_handlers() try: tracing_dir = None reporter.build_id = build_id with Tracing("main"): with BuckProject.from_current_dir() as project: tracing_dir = os.path.join(project.get_buck_out_log_dir(), "traces") with get_repo(project) as buck_repo: required_java_version = buck_repo.get_buck_compiled_java_version() java_path = get_java_path(required_java_version) _try_to_verify_java_version_off_thread( java_version_status_queue, java_path, required_java_version ) # If 'kill' is the second argument, shut down the buckd # process if argv[1:] == ["kill"]: buck_repo.kill_buckd() return ExitCode.SUCCESS return buck_repo.launch_buck(build_id, java_path, argv) finally: if tracing_dir: Tracing.write_to_dir(tracing_dir, build_id) _emit_java_version_warnings_if_any(java_version_status_queue)
def main(argv, reporter): java_version_status_queue = Queue(maxsize=1) _try_to_verify_java_version_off_thread(java_version_status_queue) def get_repo(p): # Try to detect if we're running a PEX by checking if we were invoked # via a zip file. if zipfile.is_zipfile(argv[0]): from buck_package import BuckPackage return BuckPackage(p, reporter) else: from buck_repo import BuckRepo return BuckRepo(THIS_DIR, p, reporter) # If 'killall' is the second argument, shut down all the buckd processes if sys.argv[1:] == ["killall"]: return killall_buck(reporter) install_signal_handlers() try: tracing_dir = None build_id = os.environ.get("BUCK_BUILD_ID", str(uuid.uuid4())) reporter.build_id = build_id with Tracing("main"): with BuckProject.from_current_dir() as project: tracing_dir = os.path.join(project.get_buck_out_log_dir(), "traces") with get_repo(project) as buck_repo: # If 'kill' is the second argument, shut down the buckd # process if sys.argv[1:] == ["kill"]: buck_repo.kill_buckd() return ExitCode.SUCCESS return buck_repo.launch_buck(build_id) finally: if tracing_dir: Tracing.write_to_dir(tracing_dir, build_id) _emit_java_version_warnings_if_any(java_version_status_queue)
def main(argv, reporter): java_version_status_queue = Queue(maxsize=1) _try_to_verify_java_version_off_thread(java_version_status_queue) def get_repo(p): # Try to detect if we're running a PEX by checking if we were invoked # via a zip file. if zipfile.is_zipfile(argv[0]): from buck_package import BuckPackage return BuckPackage(p, reporter) else: from buck_repo import BuckRepo return BuckRepo(THIS_DIR, p, reporter) # If 'killall' is the second argument, shut down all the buckd processes if sys.argv[1:] == ['killall']: return killall_buck(reporter) install_signal_handlers() try: tracing_dir = None build_id = os.environ.get('BUCK_BUILD_ID', str(uuid.uuid4())) reporter.build_id = build_id with Tracing("main"): with BuckProject.from_current_dir() as project: tracing_dir = os.path.join(project.get_buck_out_log_dir(), 'traces') with get_repo(project) as buck_repo: # If 'kill' is the second argument, shut down the buckd # process if sys.argv[1:] == ['kill']: buck_repo.kill_buckd() return ExitCode.SUCCESS return buck_repo.launch_buck(build_id) finally: if tracing_dir: Tracing.write_to_dir(tracing_dir, build_id) _emit_java_version_warnings_if_any(java_version_status_queue)
def setup_watchman_watch(): with Tracing('BuckTool._setup_watchman_watch'): if not which('watchman'): message = textwrap.dedent("""\ Watchman not found, please install when using buckd. See https://github.com/facebook/watchman for details.""") if sys.platform == "darwin": message += "\n(brew install watchman on OS X)" # Bail if watchman isn't installed as we know java's # FileSystemWatcher will take too long to process events. raise BuckToolException(message) logging.debug("Using watchman.")
def from_current_dir(): with Tracing("BuckProject.from_current_dir"): current_dir = os.getcwd() if "--version" in sys.argv or "-V" in sys.argv: return BuckProject(current_dir) at_root_dir = False while not at_root_dir: if os.path.exists(os.path.join(current_dir, ".buckconfig")): return BuckProject(current_dir) parent_dir = os.path.dirname(current_dir) at_root_dir = current_dir == parent_dir current_dir = parent_dir raise NoBuckConfigFoundException()
def _get_extra_java_args(self): with Tracing("BuckRepo._get_extra_java_args"): modules_dir = os.path.join(self.buck_dir, "ant-out", "buck-modules") module_resources_dir = os.path.join(self.buck_dir, "ant-out", "buck-modules-resources") return [ "-Dbuck.git_dirty={0}".format(int( self._get_buck_repo_dirty())), "-Dpf4j.pluginsDir={0}".format(modules_dir), "-Dbuck.mode=repository", "-Dbuck.module.resources={0}".format(module_resources_dir), ]
def kill_buckd(self): with Tracing('BuckTool.kill_buckd'): buckd_transport_file_path = self._buck_project.get_buckd_transport_file_path( ) if transport_exists(buckd_transport_file_path): buckd_pid = self._buck_project.get_running_buckd_pid() logging.debug("Shutting down buck daemon.") wait_socket_close = False try: with NailgunConnection( self._buck_project.get_buckd_transport_address(), cwd=self._buck_project.root) as c: c.send_command('ng-stop') wait_socket_close = True except NailgunException as e: if e.code not in (NailgunException.CONNECT_FAILED, NailgunException.CONNECTION_BROKEN, NailgunException.UNEXPECTED_CHUNKTYPE): raise BuckToolException( 'Unexpected error shutting down nailgun server: ' + str(e)) # If ng-stop command succeeds, wait for buckd process to terminate and for the # socket to close. On Unix ng-stop always drops the connection and throws. if wait_socket_close: for i in range(0, 300): if not transport_exists(buckd_transport_file_path): break time.sleep(0.01) elif buckd_pid is not None and os.name == 'posix': # otherwise just wait for up to 5 secs for the process to die # TODO(buck_team) implement wait for process and hard kill for Windows too if not wait_for_process_posix(buckd_pid, 5000): # There is a possibility that daemon is dead for some time but pid file # still exists and another process is assigned to the same pid. Ideally we # should check first which process we are killing but so far let's pretend # this will never happen and kill it with fire anyways. try: force_kill_process_posix(buckd_pid) except Exception as e: # In the worst case it keeps running multiple daemons simultaneously # consuming memory. Not the good place to be, but let's just issue a # warning for now. logging.warning( 'Error killing running Buck daemon ' + str(e)) if transport_exists(buckd_transport_file_path): force_close_transport(buckd_transport_file_path) self._buck_project.clean_up_buckd()
def _execute_command_and_maybe_run_target(self, run_fn, env, argv, java11_test_mode): """ Run a buck command using the specified `run_fn`. If the command is "run", get the path, args, etc. from the daemon, and raise an exception that tells __main__ to run that binary """ with Tracing("buck", args={"command": sys.argv[1:]}): if os.name == "nt": # Windows now support the VT100 sequences that are used by the Superconsole, but our # process must opt into this behavior to enable it. import ctypes kernel32 = ctypes.windll.kernel32 STD_OUTPUT_HANDLE = -11 ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x0004 # These calls can fail because we're not output to a terminal or we're not on a new # enough version of Windows. That's fine because we fail silently, and the # Superconsole wasn't going to work anyways. handle = kernel32.GetStdHandle(STD_OUTPUT_HANDLE) console_mode = ctypes.c_uint(0) if kernel32.GetConsoleMode(handle, ctypes.byref(console_mode)): kernel32.SetConsoleMode( handle, console_mode.value | ENABLE_VIRTUAL_TERMINAL_PROCESSING) argv = argv[1:] if len(argv) == 0 or argv[0] != "run": return run_fn(argv, env, java11_test_mode) else: with tempfile.NamedTemporaryFile( dir=self._tmp_dir) as argsfile: # Splice in location of command file to run outside buckd argv = [argv[0]] + ["--command-args-file", argsfile.name ] + argv[1:] exit_code = run_fn(argv, env, java11_test_mode) if exit_code != 0 or os.path.getsize(argsfile.name) == 0: # Build failed, so there's nothing to run. Exit normally. return exit_code cmd = json.load(argsfile) path = cmd["path"].encode("utf8") argv = [arg.encode("utf8") for arg in cmd["argv"]] envp = { k.encode("utf8"): v.encode("utf8") for k, v in cmd["envp"].iteritems() } cwd = cmd["cwd"].encode("utf8") raise ExecuteTarget(path, argv, envp, cwd)
def main(argv, reporter): java_version = _get_java_version() if java_version and java_version != REQUIRED_JAVA_VERSION: _warn_about_wrong_java_version(REQUIRED_JAVA_VERSION, java_version) def get_repo(p): # Try to detect if we're running a PEX by checking if we were invoked # via a zip file. if zipfile.is_zipfile(argv[0]): from buck_package import BuckPackage return BuckPackage(p, reporter) else: from buck_repo import BuckRepo return BuckRepo(THIS_DIR, p, reporter) # If 'killall' is the second argument, shut down all the buckd processes if sys.argv[1:] == ['killall']: return killall_buck(reporter) install_signal_handlers() try: tracing_dir = None build_id = str(uuid.uuid4()) reporter.build_id = build_id with Tracing("main"): with BuckProject.from_current_dir() as project: tracing_dir = os.path.join(project.get_buck_out_log_dir(), 'traces') with get_repo(project) as buck_repo: # If 'kill' is the second argument, shut down the buckd # process if sys.argv[1:] == ['kill']: buck_repo.kill_buckd() return 0 return buck_repo.launch_buck(build_id) finally: if tracing_dir: Tracing.write_to_dir(tracing_dir, build_id)
def _get_java_args(self, version_uid, extra_default_options=[]): with Tracing('BuckTool._get_java_args'): java_args = [ "-Xmx{0}m".format(JAVA_MAX_HEAP_SIZE_MB), "-Djava.awt.headless=true", "-Djava.util.logging.config.class=com.facebook.buck.cli.bootstrapper.LogConfig", "-Dbuck.test_util_no_tests_dir=true", "-Dbuck.version_uid={0}".format(version_uid), "-Dbuck.buckd_dir={0}".format(self._buck_project.buckd_dir), "-Dorg.eclipse.jetty.util.log.class=org.eclipse.jetty.util.log.JavaUtilLog", ] resource_lock_path = self._get_resource_lock_path() if resource_lock_path is not None: java_args.append( "-Dbuck.resource_lock_path={0}".format(resource_lock_path)) for resource in self._get_exported_resources(): if self._has_resource(resource): java_args.append("-Dbuck.{0}={1}".format( resource.name, self._get_resource(resource))) if sys.platform == "darwin": java_args.append("-Dbuck.enable_objc=true") java_args.append("-Djava.library.path=" + os.path.dirname( self._get_resource(Resource("libjcocoa.dylib")))) if "BUCK_DEBUG_MODE" in os.environ and os.environ.get( "BUCK_DEBUG_MODE") != "0": java_args.append("-agentlib:jdwp=transport=dt_socket," "server=y,suspend=y,address=8888") if os.environ.get("BUCK_DEBUG_SOY"): java_args.append("-Dbuck.soy.debug=true") java_args.extend(extra_default_options) if self._buck_project.buck_javaargs: java_args.extend(shlex.split(self._buck_project.buck_javaargs)) if self._buck_project.buck_javaargs_local: java_args.extend( shlex.split(self._buck_project.buck_javaargs_local)) java_args.extend(self._get_extra_java_args()) extra_java_args = os.environ.get("BUCK_EXTRA_JAVA_ARGS") if extra_java_args: java_args.extend(shlex.split(extra_java_args)) return java_args
def _get_buck_version_uid(self): with Tracing('BuckRepo._get_buck_version_uid'): if self._fake_buck_version: return self._fake_buck_version # First try to get the "clean" buck version. If it succeeds, # return it. clean_version = buck_version.get_clean_buck_version( self.buck_dir, allow_dirty=self._is_buck_repo_dirty_override == "1") if clean_version is not None: return clean_version return buck_version.get_dirty_buck_version(self.buck_dir)
def main(argv): try: java_home = os.getenv("JAVA_HOME", "") path = os.getenv("PATH", "") if java_home: pathsep = os.pathsep if sys.platform == 'cygwin': pathsep = ';' os.environ["PATH"] = os.path.join(java_home, 'bin') + pathsep + path tracing_dir = None build_id = str(uuid.uuid4()) with Tracing("main"): with BuckProject.from_current_dir() as project: tracing_dir = os.path.join(project.get_buck_out_log_dir(), 'traces') # Try to detect if we're running a PEX by checking if we were invoked # via a zip file. if zipfile.is_zipfile(argv[0]): from buck_package import BuckPackage buck_repo = BuckPackage(project) else: from buck_repo import BuckRepo buck_repo = BuckRepo(THIS_DIR, project) # If 'kill' is the second argument, shut down the buckd process. if sys.argv[1:] == ['kill']: buck_repo.kill_buckd() return 0 return buck_repo.launch_buck(build_id) finally: if tracing_dir: Tracing.write_to_dir(tracing_dir, build_id)
def execute(self): should_run_res_task = self._invoker.check_res_task() if not should_run_res_task: self.debug('no need to execute') return self.debug('start to execute aapt command...') self._invoker.fill_dependant_jars() self._invoker.check_ids_change() with Tracing("generate_id_keeper_files"): self._invoker.generate_r_file() # self._invoker.backup_res_files() with Tracing("incremental_databinding_process"): self._invoker.process_databinding(self._original_changed_files, self._changed_files_ref) with Tracing("run_incremental_aapt_task"): self._invoker.run_aapt_task() with Tracing("check_other_modules_resources"): self._invoker.check_other_modules_resources() self._invoker.recover_original_file_path()
def _checkout_and_clean(self, revision, branch): with Tracing('BuckRepo._checkout_and_clean'): if not self._revision_exists(revision): print(textwrap.dedent("""\ Required revision {0} is not available in the local repository. Buck is fetching updates from git. You can disable this by creating a '.nobuckcheck' file in your repository, but this might lead to strange bugs or build failures.""".format(revision)), file=sys.stderr) git_command = ['git', 'fetch'] git_command.extend( ['--all'] if not branch else ['origin', branch]) try: subprocess.check_call(git_command, stdout=sys.stderr, cwd=self._buck_dir) except subprocess.CalledProcessError: raise BuckToolException( textwrap.dedent("""\ Failed to fetch Buck updates from git.""")) current_revision = self._get_git_revision() if current_revision != revision: print(textwrap.dedent("""\ Buck is at {0}, but should be {1}. Buck is updating itself. To disable this, add a '.nobuckcheck' file to your project root. In general, you should only disable this if you are developing Buck.""".format( current_revision, revision)), file=sys.stderr) try: subprocess.check_call( ['git', 'checkout', '--quiet', revision], cwd=self._buck_dir) except subprocess.CalledProcessError: raise BuckToolException( textwrap.dedent("""\ Failed to update Buck to revision {0}.""".format( revision))) if os.path.exists(self._build_success_file): os.remove(self._build_success_file) ant = self._check_for_ant() self._run_ant_clean(ant) raise RestartBuck()
def kill_buckd(self): with Tracing('BuckTool.kill_buckd'): buckd_socket_path = self._buck_project.get_buckd_socket_path() if os.path.exists(buckd_socket_path): print("Shutting down nailgun server...", file=sys.stderr) try: with NailgunConnection('local:.buckd/sock', cwd=self._buck_project.root) as c: c.send_command('ng-stop') except NailgunException as e: if e.code not in (NailgunException.CONNECT_FAILED, NailgunException.CONNECTION_BROKEN, NailgunException.UNEXPECTED_CHUNKTYPE): raise BuckToolException( 'Unexpected error shutting down nailgun server: ' + str(e)) self._buck_project.clean_up_buckd()
def kill_buckd(self): with Tracing('BuckTool.kill_buckd'): buckd_transport_file_path = self._buck_project.get_buckd_transport_file_path() if transport_exists(buckd_transport_file_path): logging.debug("Shutting down buck daemon.") try: with NailgunConnection(self._buck_project.get_buckd_transport_address(), cwd=self._buck_project.root) as c: c.send_command('ng-stop') except NailgunException as e: if e.code not in (NailgunException.CONNECT_FAILED, NailgunException.CONNECTION_BROKEN, NailgunException.UNEXPECTED_CHUNKTYPE): raise BuckToolException( 'Unexpected error shutting down nailgun server: ' + str(e)) self._buck_project.clean_up_buckd()
def tracing(self, numDays, barcode=None): if not barcode: return self.index(error="No member selected") self.checkPermissions() with self.dbConnect() as dbConnection: dictVisits = Tracing().getDictVisits(dbConnection, barcode, numDays) (_, displayName) = self.engine.members.getName( dbConnection, barcode) if not displayName: (_, displayName) = self.engine.guests.getName( dbConnection, barcode) return self.template('tracing.mako', displayName=displayName, dictVisits=dictVisits, error="")
def _get_buck_version_uid(self): with Tracing('BuckRepo._get_buck_version_uid'): # First try to get the "clean" buck version. If it succeeds, # return it. clean_version = buck_version.get_clean_buck_version( self._buck_dir, allow_dirty=self._is_buck_repo_dirty_override == "1") if clean_version is not None: return clean_version # Otherwise, if there is a .nobuckcheck file, or if there isn't # a .buckversion file, fall back to a "dirty" version. if (self._buck_project.has_no_buck_check or not self._buck_project.buck_version): return buck_version.get_dirty_buck_version(self._buck_dir) if self._has_local_changes(): print(textwrap.dedent("""\ ::: Your buck directory has local modifications, and therefore ::: builds will not be able to use a distributed cache. ::: The following files must be either reverted or committed:""" ), file=sys.stderr) subprocess.call(['git', 'ls-files', '-m'], stdout=sys.stderr, cwd=self._buck_dir) elif os.environ.get('BUCK_CLEAN_REPO_IF_DIRTY') != 'NO': print(textwrap.dedent("""\ ::: Your local buck directory is dirty, and therefore builds will ::: not be able to use a distributed cache."""), file=sys.stderr) if sys.stdout.isatty(): print( "::: Do you want to clean your buck directory? [y/N]", file=sys.stderr) choice = raw_input().lower() if choice == "y": subprocess.call(['git', 'clean', '-fd'], stdout=sys.stderr, cwd=self._buck_dir) raise RestartBuck() return buck_version.get_dirty_buck_version(self._buck_dir)
def _is_buckd_running(self): with Tracing('BuckRepo._is_buckd_running'): buckd_socket_path = self._buck_project.get_buckd_socket_path() if not os.path.exists(buckd_socket_path): return False try: with NailgunConnection('local:.buckd/sock', stdin=None, stdout=None, stderr=None) as c: c.send_command('ng-stats') except NailgunException as e: if e.code == NailgunException.CONNECT_FAILED: return False else: raise return True
def launch_buck(self, build_id): with Tracing('BuckTool.launch_buck'): with JvmCrashLogger(self, self._buck_project.root): if self._command_line.command == "clean" and \ not self._command_line.is_help(): self.kill_buckd() buck_version_uid = self._get_buck_version_uid() if self._command_line.is_version(): print("buck version {}".format(buck_version_uid)) return 0 use_buckd = self._use_buckd if not self._command_line.is_help(): has_watchman = bool(which('watchman')) if use_buckd and has_watchman: running_version = self._buck_project.get_running_buckd_version( ) if running_version != buck_version_uid: self.kill_buckd() if not self._is_buckd_running(): self.launch_buckd( buck_version_uid=buck_version_uid) elif use_buckd and not has_watchman: logging.warning( "Not using buckd because watchman isn't installed." ) elif not use_buckd: logging.warning( "Not using buckd because NO_BUCKD is set.") env = self._environ_for_buck() env['BUCK_BUILD_ID'] = build_id use_nailgun = use_buckd and self._is_buckd_running() run_fn = self._run_with_nailgun if use_nailgun else self._run_without_nailgun self._unpack_modules() return self._execute_command_and_maybe_run_target(run_fn, env)
def _setup_watchman_watch(self): with Tracing('BuckRepo._setup_watchman_watch'): if not which('watchman'): message = textwrap.dedent("""\ Watchman not found, please install when using buckd. See https://github.com/facebook/watchman for details.""") if sys.platform == "darwin": message += "\n(brew install watchman on OS X)" # Bail if watchman isn't installed as we know java's # FileSystemWatcher will take too long to process events. raise BuckToolException(message) print("Using watchman.", file=sys.stderr) try: check_output(['watchman', 'watch', self._buck_project.root], stderr=subprocess.STDOUT) except CalledProcessError as e: print(e.output, end='', file=sys.stderr) raise
def _is_buckd_running(self): with Tracing('BuckTool._is_buckd_running'): transport_file_path = self._buck_project.get_buckd_transport_file_path() if not transport_exists(transport_file_path): return False try: with NailgunConnection( self._buck_project.get_buckd_transport_address(), stdin=None, stdout=None, stderr=None, cwd=self._buck_project.root) as c: c.send_command('ng-stats') except NailgunException as e: if e.code == NailgunException.CONNECT_FAILED: return False else: raise return True
def _get_buck_version_uid(self): with Tracing('BuckRepo._get_buck_version_uid'): if not self._is_git: return 'N/A' if not self._is_dirty(): return self._get_git_revision() if (self._buck_project.has_no_buck_check or not self._buck_project.buck_version): return self._compute_local_hash() if self._has_local_changes(): print(textwrap.dedent("""\ ::: Your buck directory has local modifications, and therefore ::: builds will not be able to use a distributed cache. ::: The following files must be either reverted or committed:""" ), file=sys.stderr) subprocess.call(['git', 'ls-files', '-m'], stdout=sys.stderr, cwd=self._buck_dir) elif os.environ.get('BUCK_CLEAN_REPO_IF_DIRTY') != 'NO': print(textwrap.dedent("""\ ::: Your local buck directory is dirty, and therefore builds will ::: not be able to use a distributed cache."""), file=sys.stderr) if sys.stdout.isatty(): print( "::: Do you want to clean your buck directory? [y/N]", file=sys.stderr) choice = raw_input().lower() if choice == "y": subprocess.call(['git', 'clean', '-fd'], stdout=sys.stderr, cwd=self._buck_dir) raise RestartBuck() return self._compute_local_hash()
def _run_without_nailgun(self, argv, env): """ Run the command by directly invoking `java` (rather than by sending a command via nailgun) """ command = ["buck"] extra_default_options = [ "-Djava.io.tmpdir={0}".format(self._tmp_dir), "-Dfile.encoding=UTF-8", "-XX:SoftRefLRUPolicyMSPerMB=0", "-XX:+UseG1GC", ] command.extend(self._get_java_args(self._get_buck_version_uid(), extra_default_options)) command.append("com.facebook.buck.cli.bootstrapper.ClassLoaderBootstrapper") command.append("com.facebook.buck.cli.Main") command.extend(self._add_args_from_env(argv)) now = int(round(time.time() * 1000)) env['BUCK_PYTHON_SPACE_INIT_TIME'] = str(now - self._init_timestamp) java = get_java_path() with Tracing('buck', args={'command': command}): return subprocess.call(command, cwd=self._buck_project.root, env=env, executable=java)
def _execute_command_and_maybe_run_target(self, run_fn, env): """ Run a buck command using the specified `run_fn`. If the command is "run", get the path, args, etc. from the daemon, and raise an exception that tells __main__ to run that binary """ with Tracing('buck', args={'command': sys.argv[1:]}): argv = sys.argv[1:] if len(argv) == 0 or argv[0] != 'run': return run_fn(argv, env) else: with tempfile.NamedTemporaryFile(dir=self._tmp_dir) as argsfile: # Splice in location of command file to run outside buckd argv = [argv[0]] + ['--command-args-file', argsfile.name] + argv[1:] exit_code = run_fn(argv, env) if exit_code != 0 or os.path.getsize(argsfile.name) == 0: # Build failed, so there's nothing to run. Exit normally. return exit_code cmd = json.load(argsfile) path = cmd['path'].encode('utf8') argv = [arg.encode('utf8') for arg in cmd['argv']] envp = {k.encode('utf8'): v.encode('utf8') for k, v in cmd['envp'].iteritems()} cwd = cmd['cwd'].encode('utf8') raise ExecuteTarget(path, argv, envp, cwd)
def _is_buckd_running(self): with Tracing('BuckRepo._is_buckd_running'): buckd_port = self._buck_project.get_buckd_port() if buckd_port is None or not buckd_port.isdigit(): return False buck_client_file = self._get_resource(CLIENT) command = [buck_client_file] command.append('ng-stats') command.append('--nailgun-port') command.append(buckd_port) try: check_output(command, cwd=self._buck_project.root, stderr=subprocess.STDOUT) except CalledProcessError as e: if e.returncode == NAILGUN_CONNECTION_REFUSED_CODE: return False else: print(e.output, end='', file=sys.stderr) raise return True
def _is_buckd_running(self): with Tracing('BuckRepo._is_buckd_running'): buckd_socket_path = self._buck_project.get_buckd_socket_path() if not os.path.exists(buckd_socket_path): return False buck_client_file = self._get_resource(CLIENT) command = [buck_client_file] command.append('ng-stats') command.append('--nailgun-server') command.append('local:.buckd/sock') try: check_output(command, cwd=self._buck_project.root, stderr=subprocess.STDOUT) except CalledProcessError as e: if e.returncode == NAILGUN_CONNECTION_REFUSED_CODE: return False else: print(e.output, end='', file=sys.stderr) raise return True
def kill_buckd(self): with Tracing('BuckRepo.kill_buckd'): buckd_socket_path = self._buck_project.get_buckd_socket_path() if os.path.exists(buckd_socket_path): print("Shutting down nailgun server...", file=sys.stderr) buck_client_file = self._get_resource(CLIENT) command = [buck_client_file] command.append('ng-stop') command.append('--nailgun-server') command.append("local:.buckd/sock") try: check_output( command, cwd=self._buck_project.root, stderr=subprocess.STDOUT) except CalledProcessError as e: if (e.returncode not in [NAILGUN_CONNECTION_REFUSED_CODE, NAILGUN_CONNECTION_BROKEN_CODE, NAILGUN_UNEXPECTED_CHUNK_TYPE]): print(e.output, end='', file=sys.stderr) raise self._buck_project.clean_up_buckd()
def launch_buck(self, build_id): with Tracing('BuckRepo.launch_buck'): self.kill_autobuild() if 'clean' in sys.argv: self.kill_buckd() buck_version_uid = self._get_buck_version_uid() use_buckd = self._use_buckd() has_watchman = bool(which('watchman')) if use_buckd and has_watchman: buckd_run_count = self._buck_project.get_buckd_run_count() running_version = self._buck_project.get_running_buckd_version( ) new_buckd_run_count = buckd_run_count + 1 if (buckd_run_count == MAX_BUCKD_RUN_COUNT or running_version != buck_version_uid): self.kill_buckd() new_buckd_run_count = 0 if new_buckd_run_count == 0 or not self._is_buckd_running(): self.launch_buckd(buck_version_uid=buck_version_uid) else: self._buck_project.update_buckd_run_count( new_buckd_run_count) elif use_buckd and not has_watchman: print("Not using buckd because watchman isn't installed.", file=sys.stderr) env = self._environ_for_buck() env['BUCK_BUILD_ID'] = build_id buck_client_file = self._get_resource(CLIENT) if use_buckd and self._is_buckd_running() and os.path.exists( buck_client_file): print("Using buckd.", file=sys.stderr) buckd_port = self._buck_project.get_buckd_port() if not buckd_port or not buckd_port.isdigit(): print( "Daemon port file is corrupt, starting new buck process.", file=sys.stderr) self.kill_buckd() else: command = [buck_client_file] command.append("--nailgun-port") command.append(buckd_port) command.append("com.facebook.buck.cli.Main") command.extend(sys.argv[1:]) with Tracing('buck', args={'command': command}): exit_code = subprocess.call( command, cwd=self._buck_project.root, env=env) if exit_code == 2: print('Daemon is busy, please wait', 'or run "buck kill" to terminate it.', file=sys.stderr) return exit_code command = ["buck"] command.extend(self._get_java_args(buck_version_uid)) command.append("-Djava.io.tmpdir={0}".format(self._tmp_dir)) command.append( "com.facebook.buck.cli.bootstrapper.ClassLoaderBootstrapper") command.append("com.facebook.buck.cli.Main") command.extend(sys.argv[1:]) return subprocess.call(command, cwd=self._buck_project.root, env=env, executable=which("java"))
def launch_buckd(self, buck_version_uid=None): with Tracing('BuckRepo.launch_buckd'): self._setup_watchman_watch() if buck_version_uid is None: buck_version_uid = self._get_buck_version_uid() # Override self._tmp_dir to a long lived directory. buckd_tmp_dir = self._buck_project.create_buckd_tmp_dir() ngserver_output_path = os.path.join(buckd_tmp_dir, 'ngserver-out') ''' Use SoftRefLRUPolicyMSPerMB for immediate GC of javac output. Set timeout to 60s (longer than the biggest GC pause seen for a 2GB heap) and GC target to 15s. This means that the GC has to miss its target by 100% or many 500ms heartbeats must be missed before a client disconnection occurs. Specify port 0 to allow Nailgun to find an available port, then parse the port number out of the first log entry. ''' command = ["buckd"] command.extend(self._get_java_args(buck_version_uid)) command.append("-Dbuck.buckd_launch_time_nanos={0}".format( monotonic_time_nanos())) command.append( "-XX:MaxGCPauseMillis={0}".format(GC_MAX_PAUSE_TARGET)) command.append("-XX:SoftRefLRUPolicyMSPerMB=0") command.append("-Djava.io.tmpdir={0}".format(buckd_tmp_dir)) command.append( "-Dcom.martiansoftware.nailgun.NGServer.outputPath={0}".format( ngserver_output_path)) command.append( "com.facebook.buck.cli.bootstrapper.ClassLoaderBootstrapper") command.append("com.martiansoftware.nailgun.NGServer") command.append("localhost:0") command.append("{0}".format(BUCKD_CLIENT_TIMEOUT_MILLIS)) ''' Change the process group of the child buckd process so that when this script is interrupted, it does not kill buckd. ''' def preexec_func(): # Close any open file descriptors to further separate buckd from its # invoking context (e.g. otherwise we'd hang when running things like # `ssh localhost buck clean`). # N.B. preexec_func is POSIX-only, and any reasonable # POSIX system has a /dev/null os.setpgrp() dev_null_fd = os.open("/dev/null", os.O_RDWR) os.dup2(dev_null_fd, 0) os.dup2(dev_null_fd, 1) os.dup2(dev_null_fd, 2) os.close(dev_null_fd) process = subprocess.Popen(command, executable=which("java"), cwd=self._buck_project.root, close_fds=True, preexec_fn=preexec_func, env=self._environ_for_buck()) buckd_port = None for i in range(100): if buckd_port: break try: with open(ngserver_output_path) as f: for line in f: match = BUCKD_LOG_FILE_PATTERN.match(line) if match: buckd_port = match.group(1) break except IOError as e: pass finally: time.sleep(0.1) else: print( "nailgun server did not respond after 10s. Aborting buckd.", file=sys.stderr) return self._buck_project.save_buckd_port(buckd_port) self._buck_project.save_buckd_version(buck_version_uid) self._buck_project.update_buckd_run_count(0)
def write_contents_to_file(path, contents): with Tracing("BuckProject.write_contents_to_file", args={"path": path}): with open(path, "w") as output_file: output_file.write(str(contents))
def clean_up_buckd(self): with Tracing("BuckProject.clean_up_buckd"): if os.path.exists(self.buckd_dir): file_locks.rmtree_if_can_lock(self.buckd_dir)