def _make_data_provider(self):
"""Returns `(data_provider, deprecated_multiplexer)`."""
flags = self.flags
if flags.grpc_data_provider:
ingester = server_ingester.ExistingServerDataIngester(
flags.grpc_data_provider,
channel_creds_type=flags.grpc_creds_type,
)
elif flags.load_fast:
ingester = server_ingester.SubprocessServerDataIngester(
logdir=flags.logdir,
reload_interval=flags.reload_interval,
channel_creds_type=flags.grpc_creds_type,
)
else:
ingester = local_ingester.LocalDataIngester(flags)
# Stash ingester so that it can avoid GCing Windows file handles.
# (See comment in `SubprocessServerDataIngester.start` for details.)
self._ingester = ingester
ingester.start()
deprecated_multiplexer = None
if isinstance(ingester, local_ingester.LocalDataIngester):
deprecated_multiplexer = ingester.deprecated_multiplexer
return (ingester.data_provider, deprecated_multiplexer)
def test(self):
# Create a fake server binary so that the `os.path.exists` check
# passes.
fake_binary = os.path.join(self.get_temp_dir(), "server")
with open(fake_binary, "wb"):
pass
self.enter_context(
mock.patch.dict(
os.environ,
{server_ingester._ENV_DATA_SERVER_BINARY: fake_binary},
))
real_popen = subprocess.Popen
port_file = None # value of `--port-file` to be stashed here
# Stub out `subprocess.Popen` to write the port file.
def fake_popen(subprocess_args, *args, **kwargs):
def target():
time.sleep(0.2) # wait one cycle
for arg in subprocess_args:
port_file_prefix = "--port-file="
if not arg.startswith(port_file_prefix):
continue
nonlocal port_file
port_file = arg[len(port_file_prefix):]
with open(port_file, "w") as outfile:
outfile.write("23456\n")
result = mock.create_autospec(real_popen, instance=True)
result.stdin = mock.Mock()
result.poll = lambda: None
result.pid = 789
threading.Thread(target=target).start()
return result
with mock.patch.object(subprocess, "Popen", wraps=fake_popen) as popen:
with mock.patch.object(grpc, "secure_channel",
autospec=True) as sc:
logdir = "/tmp/logs"
ingester = server_ingester.SubprocessServerDataIngester(
logdir=logdir,
reload_interval=5,
)
ingester.start()
self.assertIsInstance(ingester.data_provider,
grpc_provider.GrpcDataProvider)
expected_args = [
fake_binary,
"--logdir=/tmp/logs",
"--reload=5",
"--port=0",
"--port-file=%s" % port_file,
"--die-after-stdin",
"--verbose", # logging is enabled in tests
]
popen.assert_called_once_with(expected_args, stdin=subprocess.PIPE)
sc.assert_called_once_with("localhost:23456",
mock.ANY,
options=mock.ANY)
def _make_data_ingester(self):
flags = self.flags
if flags.grpc_data_provider:
return server_ingester.ExistingServerDataIngester(
flags.grpc_data_provider,
channel_creds_type=flags.grpc_creds_type,
)
if _should_use_data_server(flags.load_fast, flags.logdir):
try:
server_binary = server_ingester.get_server_binary()
except server_ingester.NoDataServerError as e:
if flags.load_fast == "true":
msg = "Option --load_fast=true not available: %s\n" % e
sys.stderr.write(msg)
sys.exit(1)
logger.info("No data server: %s", e)
else:
if flags.load_fast == "auto":
sys.stderr.write(_DATA_SERVER_MESSAGE)
sys.stderr.flush()
return server_ingester.SubprocessServerDataIngester(
server_binary=server_binary,
logdir=flags.logdir,
reload_interval=flags.reload_interval,
channel_creds_type=flags.grpc_creds_type,
samples_per_plugin=flags.samples_per_plugin,
)
return local_ingester.LocalDataIngester(flags)
def _start_subprocess_data_ingester(self):
"""Creates, starts, and returns a `SubprocessServerDataIngester`."""
flags = self.flags
server_binary = server_ingester.get_server_binary()
ingester = server_ingester.SubprocessServerDataIngester(
server_binary=server_binary,
logdir=flags.logdir,
reload_interval=flags.reload_interval,
channel_creds_type=flags.grpc_creds_type,
samples_per_plugin=flags.samples_per_plugin,
)
ingester.start()
return ingester
def test(self):
# Create a fake server binary so that the `os.path.exists` check
# passes.
fake_binary_path = os.path.join(self.get_temp_dir(), "server")
with open(fake_binary_path, "wb"):
pass
binary_info = server_ingester.ServerBinary(fake_binary_path,
version=None)
tmpdir = tempfile.TemporaryDirectory()
self.enter_context(
mock.patch.object(tempfile,
"TemporaryDirectory",
return_value=tmpdir))
port_file = os.path.join(tmpdir.name, "port")
error_file = os.path.join(tmpdir.name, "startup_error")
real_popen = subprocess.Popen
# Stub out `subprocess.Popen` to write the port file.
def fake_popen(subprocess_args, *args, **kwargs):
def target():
time.sleep(0.2) # wait one cycle
with open(port_file, "w") as outfile:
outfile.write("23456\n")
result = mock.create_autospec(real_popen, instance=True)
result.stdin = mock.Mock()
result.poll = lambda: None
result.pid = 789
threading.Thread(target=target).start()
return result
tilde_logdir = "~/tmp/logs"
expanded_logdir = os.path.expanduser(tilde_logdir)
self.assertNotEqual(tilde_logdir, expanded_logdir)
with mock.patch.object(subprocess, "Popen", wraps=fake_popen) as popen:
with mock.patch.object(grpc, "secure_channel",
autospec=True) as sc:
ingester = server_ingester.SubprocessServerDataIngester(
server_binary=binary_info,
logdir=tilde_logdir,
reload_interval=5,
channel_creds_type=grpc_util.ChannelCredsType.LOCAL,
samples_per_plugin={
"scalars": 500,
"images": 0,
},
extra_flags=["--extra-flags", "--for-fun"],
)
ingester.start()
self.assertIsInstance(ingester.data_provider,
grpc_provider.GrpcDataProvider)
expected_args = [
fake_binary_path,
"--logdir=%s" % expanded_logdir,
"--reload=5",
"--samples-per-plugin=scalars=500,images=all",
"--port=0",
"--port-file=%s" % port_file,
"--die-after-stdin",
"--error-file=%s" % error_file,
"--verbose", # logging is enabled in tests
"--extra-flags",
"--for-fun",
]
popen.assert_called_once_with(expected_args, stdin=subprocess.PIPE)
sc.assert_called_once_with("localhost:23456",
mock.ANY,
options=mock.ANY)
def test(self):
# Create a fake server binary so that the `os.path.exists` check
# passes.
fake_binary = os.path.join(self.get_temp_dir(), "server")
with open(fake_binary, "wb"):
pass
real_popen = subprocess.Popen
port_file = None # value of `--port-file` to be stashed here
# Stub out `subprocess.Popen` to write the port file.
def fake_popen(subprocess_args, *args, **kwargs):
def target():
time.sleep(0.2) # wait one cycle
for arg in subprocess_args:
port_file_prefix = "--port-file="
if not arg.startswith(port_file_prefix):
continue
nonlocal port_file
port_file = arg[len(port_file_prefix):]
with open(port_file, "w") as outfile:
outfile.write("23456\n")
result = mock.create_autospec(real_popen, instance=True)
result.stdin = mock.Mock()
result.poll = lambda: None
result.pid = 789
threading.Thread(target=target).start()
return result
tilde_logdir = "~/tmp/logs"
expanded_logdir = os.path.expanduser(tilde_logdir)
self.assertNotEqual(tilde_logdir, expanded_logdir)
with mock.patch.object(subprocess, "Popen", wraps=fake_popen) as popen:
with mock.patch.object(grpc, "secure_channel",
autospec=True) as sc:
ingester = server_ingester.SubprocessServerDataIngester(
server_binary=fake_binary,
logdir=tilde_logdir,
reload_interval=5,
channel_creds_type=grpc_util.ChannelCredsType.LOCAL,
samples_per_plugin={
"scalars": 500,
"images": 0,
},
)
ingester.start()
self.assertIsInstance(ingester.data_provider,
grpc_provider.GrpcDataProvider)
expected_args = [
fake_binary,
"--logdir=%s" % expanded_logdir,
"--reload=5",
"--port=0",
"--port-file=%s" % port_file,
"--die-after-stdin",
"--samples-per-plugin=scalars=500,images=all",
"--verbose", # logging is enabled in tests
]
popen.assert_called_once_with(expected_args, stdin=subprocess.PIPE)
sc.assert_called_once_with("localhost:23456",
mock.ANY,
options=mock.ANY)
