def test_truncate_lines_dual_lines():
assert (
truncate_lines(
["abcdefghijklmnop", "bcdefghijklmnop"], max_line_len=5, max_lines=3
)
== "ab...\nbc..."
)
def __init__(
self,
local_service_binary: Path,
port_init_max_seconds: float,
rpc_init_max_seconds: float,
process_exit_max_seconds: float,
script_args: List[str],
script_env: Dict[str, str],
):
"""Constructor.
:param local_service_binary: The path of the service binary.
:raises TimeoutError: If fails to establish connection within a specified time limit.
"""
self.process_exit_max_seconds = process_exit_max_seconds
if not Path(local_service_binary).is_file():
raise FileNotFoundError(f"File not found: {local_service_binary}")
self.cache = ServiceCache()
# The command that will be executed. The working directory of this
# command will be set to the local_service_binary's parent, so we can
# use the relpath for a neater `ps aux` view.
cmd = [
f"./{local_service_binary.name}",
f"--working_dir={self.cache.path}",
]
# Add any custom arguments
cmd += script_args
# Set the root of the runfiles directory.
env = os.environ.copy()
env["COMPILER_GYM_RUNFILES"] = str(runfiles_path("."))
env["COMPILER_GYM_SITE_DATA"] = str(site_data_path("."))
# Set the pythonpath so that executable python scripts can use absolute
# import paths like `from compiler_gym.envs.foo import bar`.
if "PYTHONPATH" in env:
env["PYTHONPATH"] = f'{env["PYTHONPATH"]}:{env["COMPILER_GYM_RUNFILES"]}'
else:
env["PYTHONPATH"] = env["COMPILER_GYM_RUNFILES"]
# Set the verbosity of the service. The logging level of the service is
# the debug level - 1, so that COMPILER_GYM_DEBUG=3 will cause VLOG(2)
# and lower to be logged to stdout.
debug_level = max(
get_debug_level(),
logging_level_to_debug_level(logger.getEffectiveLevel()))
if debug_level > 0:
cmd.append("--alsologtostderr")
cmd.append(f"-v={debug_level - 1}")
# If we are debugging the backend, set the logbuflevel to a low
# value to disable buffering of logging messages. This removes any
# buffering between `LOG(INFO) << "..."` and the message being
# emited to stderr.
cmd.append("--logbuflevel=-1")
else:
# Silence the gRPC logs as we will do our own error reporting, but
# don't override any existing value so that the user may debug the
# gRPC backend by setting GRPC_VERBOSITY to ERROR, INFO, or DEBUG.
if not os.environ.get("GRPC_VERBOSITY"):
env["GRPC_VERBOSITY"] = "NONE"
# Set environment variable COMPILER_GYM_SERVICE_ARGS to pass
# additional arguments to the service.
args = os.environ.get("COMPILER_GYM_SERVICE_ARGS", "")
if args:
cmd.append(args)
# Add any custom environment variables
env.update(script_env)
logger.debug(
"Exec `%s%s`",
" ".join(f"{k}={v}" for k, v in script_env.items()) +
" " if script_env else "",
join_cmd(cmd),
)
self.process = subprocess.Popen(
cmd,
env=env,
cwd=local_service_binary.parent,
)
self._process_returncode_exception_raised = False
# Read the port from a file generated by the service.
wait_secs = 0.1
port_path = self.cache / "port.txt"
end_time = time() + port_init_max_seconds
while time() < end_time:
returncode = self.process.poll()
if returncode is not None:
try:
# Try and decode the name of a signal. Signal returncodes
# are negative.
returncode = f"{returncode} ({Signals(abs(returncode)).name})"
except ValueError:
pass
msg = f"Service terminated with returncode: {returncode}"
# Attach any logs from the service if available.
logs = truncate_lines(self.loglines(),
max_line_len=100,
max_lines=25,
tail=True)
if logs:
msg = f"{msg}\nService logs:\n{logs}"
self.cache.close()
raise ServiceError(msg)
if port_path.is_file():
try:
with open(port_path) as f:
self.port = int(f.read().rstrip())
break
except ValueError:
# ValueError is raised by int(...) on invalid input. In that
# case, wait for longer.
pass
sleep(wait_secs)
wait_secs *= 1.2
else:
# kill() was added in Python 3.7.
if sys.version_info >= (3, 7, 0):
self.process.kill()
else:
self.process.terminate()
self.process.communicate(timeout=rpc_init_max_seconds)
self.cache.close()
raise TimeoutError("Service failed to produce port file after "
f"{port_init_max_seconds:.1f} seconds")
url = f"localhost:{self.port}"
wait_secs = 0.1
attempts = 0
end_time = time() + rpc_init_max_seconds
while time() < end_time:
try:
channel = grpc.insecure_channel(
url,
options=GRPC_CHANNEL_OPTIONS,
)
channel_ready = grpc.channel_ready_future(channel)
attempts += 1
channel_ready.result(timeout=wait_secs)
break
except (grpc.FutureTimeoutError, grpc.RpcError) as e:
logger.debug("Connection attempt %d = %s %s", attempts,
type(e).__name__, str(e))
wait_secs *= 1.2
else:
# kill() was added in Python 3.7.
if sys.version_info >= (3, 7, 0):
self.process.kill()
else:
self.process.terminate()
self.process.communicate(timeout=process_exit_max_seconds)
# Include the last few lines of logs generated by the compiler
# service, if any.
logs = truncate_lines(self.loglines(),
max_line_len=100,
max_lines=25,
tail=True)
logs_message = f" Service logs:\n{logs}" if logs else ""
self.cache.close()
raise TimeoutError(
"Failed to connect to RPC service after "
f"{rpc_init_max_seconds:.1f} seconds.{logs_message}")
super().__init__(channel, url)
def test_truncate_lines_dual_lines_generator():
def gen():
yield "abcdefghijklmnop"
yield "bcdefghijklmnop"
assert truncate_lines(gen(), max_line_len=5, max_lines=3) == "ab...\nbc..."
def test_truncate_lines_single_line():
assert truncate_lines(["abcdefghijklmnop"], max_line_len=5) == "ab..."
def test_truncate_lines_no_truncation():
assert truncate_lines(["abc"]) == "abc"
assert (
truncate_lines(["abcdef", "abcdef"], max_line_len=7, max_lines=2)
== "abcdef\nabcdef"
)