def _make_data_provider(self): """Returns `(data_provider, deprecated_multiplexer)`.""" flags = self.flags if flags.grpc_data_provider: ingester = server_ingester.ExistingServerDataIngester( flags.grpc_data_provider, channel_creds_type=flags.grpc_creds_type, ) elif flags.load_fast: ingester = server_ingester.SubprocessServerDataIngester( logdir=flags.logdir, reload_interval=flags.reload_interval, channel_creds_type=flags.grpc_creds_type, ) else: ingester = local_ingester.LocalDataIngester(flags) # Stash ingester so that it can avoid GCing Windows file handles. # (See comment in `SubprocessServerDataIngester.start` for details.) self._ingester = ingester ingester.start() deprecated_multiplexer = None if isinstance(ingester, local_ingester.LocalDataIngester): deprecated_multiplexer = ingester.deprecated_multiplexer return (ingester.data_provider, deprecated_multiplexer)
def test(self): # Create a fake server binary so that the `os.path.exists` check # passes. fake_binary = os.path.join(self.get_temp_dir(), "server") with open(fake_binary, "wb"): pass self.enter_context( mock.patch.dict( os.environ, {server_ingester._ENV_DATA_SERVER_BINARY: fake_binary}, )) real_popen = subprocess.Popen port_file = None # value of `--port-file` to be stashed here # Stub out `subprocess.Popen` to write the port file. def fake_popen(subprocess_args, *args, **kwargs): def target(): time.sleep(0.2) # wait one cycle for arg in subprocess_args: port_file_prefix = "--port-file=" if not arg.startswith(port_file_prefix): continue nonlocal port_file port_file = arg[len(port_file_prefix):] with open(port_file, "w") as outfile: outfile.write("23456\n") result = mock.create_autospec(real_popen, instance=True) result.stdin = mock.Mock() result.poll = lambda: None result.pid = 789 threading.Thread(target=target).start() return result with mock.patch.object(subprocess, "Popen", wraps=fake_popen) as popen: with mock.patch.object(grpc, "secure_channel", autospec=True) as sc: logdir = "/tmp/logs" ingester = server_ingester.SubprocessServerDataIngester( logdir=logdir, reload_interval=5, ) ingester.start() self.assertIsInstance(ingester.data_provider, grpc_provider.GrpcDataProvider) expected_args = [ fake_binary, "--logdir=/tmp/logs", "--reload=5", "--port=0", "--port-file=%s" % port_file, "--die-after-stdin", "--verbose", # logging is enabled in tests ] popen.assert_called_once_with(expected_args, stdin=subprocess.PIPE) sc.assert_called_once_with("localhost:23456", mock.ANY, options=mock.ANY)
def _make_data_ingester(self): flags = self.flags if flags.grpc_data_provider: return server_ingester.ExistingServerDataIngester( flags.grpc_data_provider, channel_creds_type=flags.grpc_creds_type, ) if _should_use_data_server(flags.load_fast, flags.logdir): try: server_binary = server_ingester.get_server_binary() except server_ingester.NoDataServerError as e: if flags.load_fast == "true": msg = "Option --load_fast=true not available: %s\n" % e sys.stderr.write(msg) sys.exit(1) logger.info("No data server: %s", e) else: if flags.load_fast == "auto": sys.stderr.write(_DATA_SERVER_MESSAGE) sys.stderr.flush() return server_ingester.SubprocessServerDataIngester( server_binary=server_binary, logdir=flags.logdir, reload_interval=flags.reload_interval, channel_creds_type=flags.grpc_creds_type, samples_per_plugin=flags.samples_per_plugin, ) return local_ingester.LocalDataIngester(flags)
def _start_subprocess_data_ingester(self): """Creates, starts, and returns a `SubprocessServerDataIngester`.""" flags = self.flags server_binary = server_ingester.get_server_binary() ingester = server_ingester.SubprocessServerDataIngester( server_binary=server_binary, logdir=flags.logdir, reload_interval=flags.reload_interval, channel_creds_type=flags.grpc_creds_type, samples_per_plugin=flags.samples_per_plugin, ) ingester.start() return ingester
def test(self): # Create a fake server binary so that the `os.path.exists` check # passes. fake_binary_path = os.path.join(self.get_temp_dir(), "server") with open(fake_binary_path, "wb"): pass binary_info = server_ingester.ServerBinary(fake_binary_path, version=None) tmpdir = tempfile.TemporaryDirectory() self.enter_context( mock.patch.object(tempfile, "TemporaryDirectory", return_value=tmpdir)) port_file = os.path.join(tmpdir.name, "port") error_file = os.path.join(tmpdir.name, "startup_error") real_popen = subprocess.Popen # Stub out `subprocess.Popen` to write the port file. def fake_popen(subprocess_args, *args, **kwargs): def target(): time.sleep(0.2) # wait one cycle with open(port_file, "w") as outfile: outfile.write("23456\n") result = mock.create_autospec(real_popen, instance=True) result.stdin = mock.Mock() result.poll = lambda: None result.pid = 789 threading.Thread(target=target).start() return result tilde_logdir = "~/tmp/logs" expanded_logdir = os.path.expanduser(tilde_logdir) self.assertNotEqual(tilde_logdir, expanded_logdir) with mock.patch.object(subprocess, "Popen", wraps=fake_popen) as popen: with mock.patch.object(grpc, "secure_channel", autospec=True) as sc: ingester = server_ingester.SubprocessServerDataIngester( server_binary=binary_info, logdir=tilde_logdir, reload_interval=5, channel_creds_type=grpc_util.ChannelCredsType.LOCAL, samples_per_plugin={ "scalars": 500, "images": 0, }, extra_flags=["--extra-flags", "--for-fun"], ) ingester.start() self.assertIsInstance(ingester.data_provider, grpc_provider.GrpcDataProvider) expected_args = [ fake_binary_path, "--logdir=%s" % expanded_logdir, "--reload=5", "--samples-per-plugin=scalars=500,images=all", "--port=0", "--port-file=%s" % port_file, "--die-after-stdin", "--error-file=%s" % error_file, "--verbose", # logging is enabled in tests "--extra-flags", "--for-fun", ] popen.assert_called_once_with(expected_args, stdin=subprocess.PIPE) sc.assert_called_once_with("localhost:23456", mock.ANY, options=mock.ANY)
def test(self): # Create a fake server binary so that the `os.path.exists` check # passes. fake_binary = os.path.join(self.get_temp_dir(), "server") with open(fake_binary, "wb"): pass real_popen = subprocess.Popen port_file = None # value of `--port-file` to be stashed here # Stub out `subprocess.Popen` to write the port file. def fake_popen(subprocess_args, *args, **kwargs): def target(): time.sleep(0.2) # wait one cycle for arg in subprocess_args: port_file_prefix = "--port-file=" if not arg.startswith(port_file_prefix): continue nonlocal port_file port_file = arg[len(port_file_prefix):] with open(port_file, "w") as outfile: outfile.write("23456\n") result = mock.create_autospec(real_popen, instance=True) result.stdin = mock.Mock() result.poll = lambda: None result.pid = 789 threading.Thread(target=target).start() return result tilde_logdir = "~/tmp/logs" expanded_logdir = os.path.expanduser(tilde_logdir) self.assertNotEqual(tilde_logdir, expanded_logdir) with mock.patch.object(subprocess, "Popen", wraps=fake_popen) as popen: with mock.patch.object(grpc, "secure_channel", autospec=True) as sc: ingester = server_ingester.SubprocessServerDataIngester( server_binary=fake_binary, logdir=tilde_logdir, reload_interval=5, channel_creds_type=grpc_util.ChannelCredsType.LOCAL, samples_per_plugin={ "scalars": 500, "images": 0, }, ) ingester.start() self.assertIsInstance(ingester.data_provider, grpc_provider.GrpcDataProvider) expected_args = [ fake_binary, "--logdir=%s" % expanded_logdir, "--reload=5", "--port=0", "--port-file=%s" % port_file, "--die-after-stdin", "--samples-per-plugin=scalars=500,images=all", "--verbose", # logging is enabled in tests ] popen.assert_called_once_with(expected_args, stdin=subprocess.PIPE) sc.assert_called_once_with("localhost:23456", mock.ANY, options=mock.ANY)