def _make_grpc_provider(self, addr):
options = [
("grpc.max_receive_message_length", 1024 * 1024 * 256),
]
channel = grpc.insecure_channel(addr, options=options)
stub = grpc_provider.make_stub(channel)
provider = grpc_provider.GrpcDataProvider(addr, stub)
return provider
def _make_provider(addr):
options = [
("grpc.max_receive_message_length", 1024 * 1024 * 256),
]
creds = grpc.local_channel_credentials()
channel = grpc.secure_channel(addr, creds, options=options)
stub = grpc_provider.make_stub(channel)
return grpc_provider.GrpcDataProvider(addr, stub)
def __init__(self, address, *, channel_creds_type):
"""Initializes an ingester with the given configuration.
Args:
address: String, as passed to `--grpc_data_provider`.
channel_creds_type: `grpc_util.ChannelCredsType`, as passed to
`--grpc_creds_type`.
"""
stub = _make_stub(address, channel_creds_type)
self._data_provider = grpc_provider.GrpcDataProvider(address, stub)
def _make_provider(addr, channel_creds_type):
(creds, options) = channel_creds_type.channel_config()
options.append(("grpc.max_receive_message_length", 1024 * 1024 * 256))
channel = grpc.secure_channel(addr, creds, options=options)
stub = grpc_provider.make_stub(channel)
return grpc_provider.GrpcDataProvider(addr, stub)
def setUp(self):
super().setUp()
self.stub = _create_mock_client()
addr = "localhost:0" # invalid, just in case it tries to connect
self.provider = grpc_provider.GrpcDataProvider(addr, self.stub)
self.ctx = context.RequestContext()
def start(self):
if self._data_provider:
return
tmpdir = tempfile.TemporaryDirectory(prefix="tensorboard_data_server_")
port_file_path = os.path.join(tmpdir.name, "port")
error_file_path = os.path.join(tmpdir.name, "startup_error")
if self._reload_interval <= 0:
reload = "once"
else:
reload = str(int(self._reload_interval))
sample_hint_pairs = [
"%s=%s" % (k, "all" if v == 0 else v)
for k, v in self._samples_per_plugin.items()
]
samples_per_plugin = ",".join(sample_hint_pairs)
args = [
self._server_binary.path,
"--logdir=%s" % os.path.expanduser(self._logdir),
"--reload=%s" % reload,
"--samples-per-plugin=%s" % samples_per_plugin,
"--port=0",
"--port-file=%s" % (port_file_path, ),
"--die-after-stdin",
]
if self._server_binary.at_least_version("0.5.0a0"):
args.append("--error-file=%s" % (error_file_path, ))
if logger.isEnabledFor(logging.INFO):
args.append("--verbose")
if logger.isEnabledFor(logging.DEBUG):
args.append("--verbose") # Repeat arg to increase verbosity.
args.extend(self._extra_flags)
logger.info("Spawning data server: %r", args)
popen = subprocess.Popen(args, stdin=subprocess.PIPE)
# Stash stdin to avoid calling its destructor: on Windows, this
# is a `subprocess.Handle` that closes itself in `__del__`,
# which would cause the data server to shut down. (This is not
# documented; you have to read CPython source to figure it out.)
# We want that to happen at end of process, but not before.
self._stdin_handle = popen.stdin # stash to avoid stdin being closed
port = None
# The server only needs about 10 microseconds to spawn on my machine,
# but give a few orders of magnitude of padding, and then poll.
time.sleep(0.01)
for i in range(20):
if popen.poll() is not None:
msg = (_maybe_read_file(error_file_path) or "").strip()
if not msg:
msg = ("exited with %d; check stderr for details" %
popen.poll())
raise DataServerStartupError(msg)
logger.info("Polling for data server port (attempt %d)", i)
port_file_contents = _maybe_read_file(port_file_path)
logger.info("Port file contents: %r", port_file_contents)
if (port_file_contents or "").endswith("\n"):
port = int(port_file_contents)
break
# Else, not done writing yet.
time.sleep(0.5)
if port is None:
raise DataServerStartupError(
"Timed out while waiting for data server to start. "
"It may still be running as pid %d." % popen.pid)
addr = "localhost:%d" % port
stub = _make_stub(addr, self._channel_creds_type)
logger.info(
"Opened channel to data server at pid %d via %s",
popen.pid,
addr,
)
req = data_provider_pb2.GetExperimentRequest()
try:
stub.GetExperiment(req, timeout=5) # should be near-instant
except grpc.RpcError as e:
msg = "Failed to communicate with data server at %s: %s" % (addr,
e)
logging.warning("%s", msg)
raise DataServerStartupError(msg) from e
logger.info("Got valid response from data server")
self._data_provider = grpc_provider.GrpcDataProvider(addr, stub)
