def _make_grpc_provider(self, addr): options = [ ("grpc.max_receive_message_length", 1024 * 1024 * 256), ] channel = grpc.insecure_channel(addr, options=options) stub = grpc_provider.make_stub(channel) provider = grpc_provider.GrpcDataProvider(addr, stub) return provider
def _make_provider(addr): options = [ ("grpc.max_receive_message_length", 1024 * 1024 * 256), ] creds = grpc.local_channel_credentials() channel = grpc.secure_channel(addr, creds, options=options) stub = grpc_provider.make_stub(channel) return grpc_provider.GrpcDataProvider(addr, stub)
def __init__(self, address, *, channel_creds_type): """Initializes an ingester with the given configuration. Args: address: String, as passed to `--grpc_data_provider`. channel_creds_type: `grpc_util.ChannelCredsType`, as passed to `--grpc_creds_type`. """ stub = _make_stub(address, channel_creds_type) self._data_provider = grpc_provider.GrpcDataProvider(address, stub)
def _make_provider(addr, channel_creds_type): (creds, options) = channel_creds_type.channel_config() options.append(("grpc.max_receive_message_length", 1024 * 1024 * 256)) channel = grpc.secure_channel(addr, creds, options=options) stub = grpc_provider.make_stub(channel) return grpc_provider.GrpcDataProvider(addr, stub)
def setUp(self): super().setUp() self.stub = _create_mock_client() addr = "localhost:0" # invalid, just in case it tries to connect self.provider = grpc_provider.GrpcDataProvider(addr, self.stub) self.ctx = context.RequestContext()
def start(self): if self._data_provider: return tmpdir = tempfile.TemporaryDirectory(prefix="tensorboard_data_server_") port_file_path = os.path.join(tmpdir.name, "port") error_file_path = os.path.join(tmpdir.name, "startup_error") if self._reload_interval <= 0: reload = "once" else: reload = str(int(self._reload_interval)) sample_hint_pairs = [ "%s=%s" % (k, "all" if v == 0 else v) for k, v in self._samples_per_plugin.items() ] samples_per_plugin = ",".join(sample_hint_pairs) args = [ self._server_binary.path, "--logdir=%s" % os.path.expanduser(self._logdir), "--reload=%s" % reload, "--samples-per-plugin=%s" % samples_per_plugin, "--port=0", "--port-file=%s" % (port_file_path, ), "--die-after-stdin", ] if self._server_binary.at_least_version("0.5.0a0"): args.append("--error-file=%s" % (error_file_path, )) if logger.isEnabledFor(logging.INFO): args.append("--verbose") if logger.isEnabledFor(logging.DEBUG): args.append("--verbose") # Repeat arg to increase verbosity. args.extend(self._extra_flags) logger.info("Spawning data server: %r", args) popen = subprocess.Popen(args, stdin=subprocess.PIPE) # Stash stdin to avoid calling its destructor: on Windows, this # is a `subprocess.Handle` that closes itself in `__del__`, # which would cause the data server to shut down. (This is not # documented; you have to read CPython source to figure it out.) # We want that to happen at end of process, but not before. self._stdin_handle = popen.stdin # stash to avoid stdin being closed port = None # The server only needs about 10 microseconds to spawn on my machine, # but give a few orders of magnitude of padding, and then poll. time.sleep(0.01) for i in range(20): if popen.poll() is not None: msg = (_maybe_read_file(error_file_path) or "").strip() if not msg: msg = ("exited with %d; check stderr for details" % popen.poll()) raise DataServerStartupError(msg) logger.info("Polling for data server port (attempt %d)", i) port_file_contents = _maybe_read_file(port_file_path) logger.info("Port file contents: %r", port_file_contents) if (port_file_contents or "").endswith("\n"): port = int(port_file_contents) break # Else, not done writing yet. time.sleep(0.5) if port is None: raise DataServerStartupError( "Timed out while waiting for data server to start. " "It may still be running as pid %d." % popen.pid) addr = "localhost:%d" % port stub = _make_stub(addr, self._channel_creds_type) logger.info( "Opened channel to data server at pid %d via %s", popen.pid, addr, ) req = data_provider_pb2.GetExperimentRequest() try: stub.GetExperiment(req, timeout=5) # should be near-instant except grpc.RpcError as e: msg = "Failed to communicate with data server at %s: %s" % (addr, e) logging.warning("%s", msg) raise DataServerStartupError(msg) from e logger.info("Got valid response from data server") self._data_provider = grpc_provider.GrpcDataProvider(addr, stub)