def test_upload_limits_provided_without_max_blob_size(self):
# This just shows that the proto3 default value of 0 is reported as
# usual, not handled as a special case.
info = server_info_pb2.ServerInfoResponse()
info.upload_limits.SetInParent()
actual = server_info.max_blob_size(info)
self.assertEqual(actual, 0)
def test_more_plugins(self):
info = server_info_pb2.ServerInfoResponse()
info.plugin_control.allowed_plugins.append("foo")
info.plugin_control.allowed_plugins.append("bar")
info.plugin_control.allowed_plugins.append("foo")
actual = server_info.allowed_plugins(info)
self.assertEqual(actual, frozenset(["foo", "bar"]))
def testUploadIntentWithExperimentUrlCallback(self):
"""Test the upload intent with a callback."""
server_info = server_info_pb2.ServerInfoResponse()
server_info.url_format.template = "https://tensorboard.dev/x/{}"
server_info.url_format.id_placeholder = "{}"
stub = dry_run_stubs.DryRunTensorBoardWriterStub()
stub.CreateExperiment = (
lambda req, **__: write_service_pb2.CreateExperimentResponse(
experiment_id="test_experiment_id", url="this URL is ignored"
)
)
expected_url = "https://tensorboard.dev/x/test_experiment_id"
with mock.patch.object(
dry_run_stubs,
"DryRunTensorBoardWriterStub",
wraps=lambda: stub,
), mock.patch.object(sys.stdout, "write"):
mock_channel = mock.Mock()
mock_experiment_url_callback = mock.Mock()
intent = uploader_subcommand.UploadIntent(
self.get_temp_dir(),
dry_run=True,
one_shot=True,
experiment_url_callback=mock_experiment_url_callback,
)
intent.execute(server_info, mock_channel)
mock_experiment_url_callback.assert_called_once_with(expected_url)
def test_upload_limits_provided_without_max_tensor_point_size(self):
# This just shows that the proto3 default value of 0 is reported as
# usual, not handled as a special case.
info = server_info_pb2.ServerInfoResponse()
# Ensure upload_limits is set but do not explicitly set
# max_tensor_point_size.
info.upload_limits.max_blob_size = 42
actual = server_info.max_tensor_point_size(info)
self.assertEqual(actual, 0)
def test_missing_max_blob_size_in_upload_limits(self):
# Test the one remaining field we did not test in
# test_missing_fields_in_upload_limits.
info = server_info_pb2.ServerInfoResponse()
info.upload_limits.max_tensor_point_size = 22
actual = server_info.upload_limits(info)
self.assertEqual(actual.max_blob_size,
server_info._DEFAULT_MAX_BLOB_SIZE)
self.assertEqual(actual.max_tensor_point_size, 22)
def app(request):
body = request.get_data()
request_pb = server_info_pb2.ServerInfoRequest.FromString(body)
self.assertEqual(request_pb.version, version.VERSION)
self.assertEqual(
request_pb.plugin_specification.upload_plugins,
["plugin1", "plugin2"],
)
return wrappers.BaseResponse(
server_info_pb2.ServerInfoResponse().SerializeToString())
def test_upload_limits_from_server_info(self):
info_upload_limits = server_info_pb2.UploadLimits()
info_upload_limits.max_scalar_request_size = 1
info_upload_limits.max_tensor_request_size = 2
info_upload_limits.max_blob_request_size = 3
info_upload_limits.min_scalar_request_interval = 4
info_upload_limits.min_tensor_request_interval = 5
info_upload_limits.min_blob_request_interval = 6
info_upload_limits.max_blob_size = 7
info_upload_limits.max_tensor_point_size = 8
info = server_info_pb2.ServerInfoResponse()
info.upload_limits.CopyFrom(info_upload_limits)
actual = server_info.upload_limits(info)
self.assertEqual(actual, info_upload_limits)
def testDeleteIntentDeletesMultipleExperiments(self):
# Setup: A uploader_lib which will report success (return `None`) on the
# deletion of an experiment.
eid_1 = "1111"
eid_2 = "22222"
eid_3 = "333333"
mock_delete_experiment = mock.MagicMock(return_value=None)
mock_stdout_write = mock.MagicMock()
# Execute: A _DeleteExperimentIntent on the list containing exactly
# three experiment_ids
#
# Mock three places:
# 1. Writing to stdout, so we can inspect messages to the user.
# 2. uploader_lib.delete_experiment itself, we will inspect invocations
# this method but do not want to actually delete anything.
# 3. The creation of the grpc TensorBoardWriterServiceStub, which
# happens in intent, but we don't want to actually open a network
# communication.
with mock.patch.object(
sys.stdout, "write", mock_stdout_write
), mock.patch.object(
uploader_lib, "delete_experiment", mock_delete_experiment
), mock.patch.object(
write_service_pb2_grpc, "TensorBoardWriterServiceStub"
):
# Set up an UploadIntent configured with one_shot and an empty temp
# directory.
intent = uploader_subcommand._DeleteExperimentIntent(
experiment_id_list=[eid_1, eid_2, eid_3]
)
# Execute the intent.execute method.
intent.execute(server_info_pb2.ServerInfoResponse(), None)
# Expect that there are three calls to delete_experiment.
self.assertEqual(mock_delete_experiment.call_count, 3)
# Expect that ".*Deleted experiment.*" and the eid are among the things
# printed.
out_msg = ",".join([x[0][0] for x in mock_stdout_write.call_args_list])
self.assertRegex(out_msg, f".*Deleted experiment {eid_1}.*")
self.assertRegex(out_msg, f".*Deleted experiment {eid_2}.*")
self.assertRegex(out_msg, f".*Deleted experiment {eid_3}.*")
def testUploadIntentOneShot(self):
"""Test the upload intent under the one-shot mode."""
# Mock three places:
# 1. The uploader itself, we will inspect invocations of its methods but
# do not want to actually uplaod anything.
# 2. Writing to stdout, so we can inspect messages to the user.
# 3. The creation of the grpc WriteServiceChannel, which happens in the
# non dry_run execution, but we don't want to actually open a network
# communication. mock_uploader = mock.MagicMock()
mock_uploader = mock.MagicMock()
mock_uploader.create_experiment = mock.MagicMock(
return_value="fake_experiment_id"
)
mock_stdout_write = mock.MagicMock()
with mock.patch.object(
sys.stdout, "write", mock_stdout_write
), mock.patch.object(
uploader_lib, "TensorBoardUploader", return_value=mock_uploader
), mock.patch.object(
write_service_pb2_grpc, "TensorBoardWriterServiceStub"
):
# Set up an UploadIntent configured with one_shot and an empty temp
# directory.
intent = uploader_subcommand.UploadIntent(
self.get_temp_dir(), one_shot=True
)
# Execute the intent.execute method.
intent.execute(server_info_pb2.ServerInfoResponse(), None)
# Expect that there is one call to create_experiment.
self.assertEqual(mock_uploader.create_experiment.call_count, 1)
# Expect that there is one call to start_uploading.
self.assertEqual(mock_uploader.start_uploading.call_count, 1)
# Expect that ".*Done scanning logdir.*" is among the things printed.
stdout_writes = [x[0][0] for x in mock_stdout_write.call_args_list]
self.assertRegex(
",".join(stdout_writes),
".*experiment created.*",
)
# Expect that the last thing written is the string "Done" and the
# experiment_id.
self.assertRegex(stdout_writes[-1], ".*Done.*")
self.assertRegex(stdout_writes[-1], ".*fake_experiment_id.*")
def test_fetches_response(self):
expected_result = server_info_pb2.ServerInfoResponse()
expected_result.compatibility.verdict = server_info_pb2.VERDICT_OK
expected_result.compatibility.details = "all clear"
expected_result.api_server.endpoint = "api.example.com:443"
expected_result.url_format.template = "http://localhost:8080/{{eid}}"
expected_result.url_format.id_placeholder = "{{eid}}"
@wrappers.BaseRequest.application
def app(request):
self.assertEqual(request.method, "POST")
self.assertEqual(request.path, "/api/uploader")
body = request.get_data()
request_pb = server_info_pb2.ServerInfoRequest.FromString(body)
self.assertEqual(request_pb.version, version.VERSION)
return wrappers.BaseResponse(expected_result.SerializeToString())
origin = self._start_server(app)
result = server_info.fetch_server_info(origin)
self.assertEqual(result, expected_result)
def create_server_info(frontend_origin, api_endpoint):
"""Manually creates server info given a frontend and backend.
Args:
frontend_origin: The origin of the TensorBoard.dev frontend, like
"https://tensorboard.dev" or "http://localhost:8000".
api_endpoint: As to `server_info_pb2.ApiServer.endpoint`.
Returns:
A `server_info_pb2.ServerInfoResponse` message.
"""
result = server_info_pb2.ServerInfoResponse()
result.compatibility.verdict = server_info_pb2.VERDICT_OK
result.api_server.endpoint = api_endpoint
url_format = result.url_format
placeholder = "{{EID}}"
while placeholder in frontend_origin:
placeholder = "{%s}" % placeholder
url_format.template = "%s/experiment/%s/" % (frontend_origin, placeholder)
url_format.id_placeholder = placeholder
return result
def test_no_upload_limits_in_server_info(self):
info = server_info_pb2.ServerInfoResponse()
actual = server_info.upload_limits(info)
expected = server_info_pb2.UploadLimits()
expected.max_scalar_request_size = (
server_info._DEFAULT_MAX_SCALAR_REQUEST_SIZE)
expected.max_tensor_request_size = (
server_info._DEFAULT_MAX_TENSOR_REQUEST_SIZE)
expected.max_blob_request_size = (
server_info._DEFAULT_MAX_BLOB_REQUEST_SIZE)
expected.min_scalar_request_interval = (
server_info._DEFAULT_MIN_SCALAR_REQUEST_INTERVAL)
expected.min_tensor_request_interval = (
server_info._DEFAULT_MIN_TENSOR_REQUEST_INTERVAL)
expected.min_blob_request_interval = (
server_info._DEFAULT_MIN_BLOB_REQUEST_INTERVAL)
expected.max_blob_size = server_info._DEFAULT_MAX_BLOB_SIZE
expected.max_tensor_point_size = (
server_info._DEFAULT_MAX_TENSOR_POINT_SIZE)
self.assertEqual(actual, expected)
def testUploadIntentOneShotEmptyDirectoryFails(self):
"""Test the upload intent under the one-shot mode with missing dir.
In the case of a non-existent directoy, uploading should not
create an experiment.
"""
# Mock three places:
# 1. The uploader itself, we will inspect invocations of its methods but
# do not want to actually uplaod anything.
# 2. Writing to stdout, so we can inspect messages to the user.
# 3. The creation of the grpc WriteServiceChannel, which happens in the
# non dry_run execution, but we don't want to actually open a network
# communication.
mock_uploader = mock.MagicMock()
mock_stdout_write = mock.MagicMock()
with mock.patch.object(
uploader_lib,
"TensorBoardUploader",
return_value=mock_uploader,
), mock.patch.object(
sys.stdout, "write", mock_stdout_write
), mock.patch.object(
write_service_pb2_grpc, "TensorBoardWriterServiceStub"
):
# Set up an UploadIntent configured with one_shot and a
# non-existent directory.
intent = uploader_subcommand.UploadIntent(
"/dev/null/non/existent/directory", one_shot=True
)
# Execute the intent.execute method.
intent.execute(server_info_pb2.ServerInfoResponse(), None)
# Expect that there is no call to create an experiment.
self.assertEqual(mock_uploader.create_experiment.call_count, 0)
# Expect a message to the user indicating no experiment was created.
stdout_writes = [x[0][0] for x in mock_stdout_write.call_args_list]
self.assertRegex(
",".join(stdout_writes),
".*Exiting without creating an experiment.*",
)
def testDeleteIntentRaisesWhenAskedToDeleteZeroExperiments(self):
mock_delete_experiment = mock.MagicMock(return_value=None)
mock_stdout_write = mock.MagicMock()
# Execute: A _DeleteExperimentIntent on the empty list.
#
# Mock three places:
# 1. Writing to stdout, so we can inspect messages to the user.
# 2. uploader_lib.delete_experiment itself, we will inspect invocations
# this method but do not want to actually delete anything.
# 3. The creation of the grpc TensorBoardWriterServiceStub, which
# happens in intent, but we don't want to actually open a network
# communication.
with mock.patch.object(
sys.stdout, "write", mock_stdout_write
), mock.patch.object(
uploader_lib, "delete_experiment", mock_delete_experiment
), mock.patch.object(
write_service_pb2_grpc, "TensorBoardWriterServiceStub"
):
# Set up an UploadIntent configured with one_shot and an empty temp
# directory.
intent = uploader_subcommand._DeleteExperimentIntent(
experiment_id_list=[]
)
# Execute the intent.execute method.
# Expect raises with appropriate message.
with self.assertRaisesRegex(
base_plugin.FlagsError,
"Must specify at least one experiment ID to delete.*",
):
intent.execute(server_info_pb2.ServerInfoResponse(), None)
# Expect:
# Expect that there are zero calls to delete_experiment.
self.assertEqual(mock_delete_experiment.call_count, 0)
def test_old_server_no_upload_limits(self):
info = server_info_pb2.ServerInfoResponse()
actual = server_info.max_tensor_point_size(info)
self.assertEqual(actual, server_info._DEFAULT_MAX_TENSOR_POINT_SIZE)
def testDeleteIntentHandlesUndeletableExperiemtns(self):
# Setup: A uploader_lib which will emit scripted results for differen
# eids. See mock_delete_func for script.
eid_1_ok = "1111"
eid_2_empty = ""
eid_3_ok = "3333"
eid_4_missing = "4444"
eid_5_ok = "5555"
eid_6_permission_denied = "6666"
eid_7_ok = "7777"
eid_8_rate_limited = "8888"
eid_9_ok = "9999"
def mock_delete_func(_, eid):
if eid == eid_4_missing:
raise uploader_lib.ExperimentNotFoundError()
if eid == eid_6_permission_denied:
raise uploader_lib.PermissionDeniedError()
if eid == eid_8_rate_limited:
raise grpc.RpcError("Rate limited")
return None
mock_delete_experiment = mock.MagicMock(side_effect=mock_delete_func)
mock_stdout_write = mock.MagicMock()
mock_stderr_write = mock.MagicMock()
mock_sys_exit = mock.MagicMock()
# Execute: A _DeleteExperimentIntent on the motley list of
# experiments.
#
# Mock five places:
# 1. Writing to stdout, so we can inspect messages to the user.
# 2. Writing to stderr.
# 3. uploader_lib.delete_experiment itself, we will inspect invocations
# this method but do not want to actually delete anything.
# 4. The creation of the grpc TensorBoardWriterServiceStub, which
# happens in intent, but we don't want to actually open a network
# communication.
# 5. Replace uploader_subcommand._die with something that wont
# actually kill the test.
with mock.patch.object(
sys.stdout, "write", mock_stdout_write
), mock.patch.object(
sys.stderr, "write", mock_stderr_write
), mock.patch.object(
uploader_lib, "delete_experiment", mock_delete_experiment
), mock.patch.object(
write_service_pb2_grpc, "TensorBoardWriterServiceStub"
), mock.patch.object(
sys, "exit", mock_sys_exit
):
# Set up an UploadIntent configured with one_shot and an empty temp
# directory.
intent = uploader_subcommand._DeleteExperimentIntent(
experiment_id_list=[
eid_1_ok,
eid_2_empty,
eid_3_ok,
eid_4_missing,
eid_5_ok,
eid_6_permission_denied,
eid_7_ok,
eid_8_rate_limited,
eid_9_ok,
]
)
# Execute the intent.execute method.
intent.execute(server_info_pb2.ServerInfoResponse(), None)
# Expect that there are eight calls to delete_experiment, one for each
# experiment *except* the empty one.
self.assertEqual(mock_delete_experiment.call_count, 8)
# Expect that ".*Deleted experiment.*" and the eid are among the things
# printed to stdout
stdout_msg = ",".join(
[x[0][0] for x in mock_stdout_write.call_args_list]
)
self.assertRegex(stdout_msg, f".*Deleted experiment {eid_1_ok}.*")
self.assertRegex(stdout_msg, f".*Deleted experiment {eid_3_ok}.*")
self.assertRegex(stdout_msg, f".*Deleted experiment {eid_5_ok}.*")
self.assertRegex(stdout_msg, f".*Deleted experiment {eid_7_ok}.*")
self.assertRegex(stdout_msg, f".*Deleted experiment {eid_9_ok}.*")
self.assertRegex(stdout_msg, ".*Skipping empty experiment_id.*")
self.assertNotRegex(
stdout_msg, f".*Deleted experiment {eid_4_missing}.*"
)
self.assertNotRegex(
stdout_msg, f".*Deleted experiment {eid_6_permission_denied}.*"
)
self.assertNotRegex(
stdout_msg, f".*Deleted experiment {eid_8_rate_limited}.*"
)
# Expect appropriate error messages sent to stderr
stderr_msg = ",".join(
[x[0][0] for x in mock_stderr_write.call_args_list]
)
self.assertRegex(stderr_msg, f".*No such experiment {eid_4_missing}.*")
self.assertRegex(
stderr_msg,
f".*Cannot delete experiment {eid_6_permission_denied}.*",
)
self.assertRegex(
stderr_msg,
f".*Internal error deleting experiment {eid_8_rate_limited}.*",
)
# Expect a call to kill the process.
self.assertEqual(mock_sys_exit.call_count, 1)
def test_scalars_only(self):
info = server_info_pb2.ServerInfoResponse()
info.plugin_control.allowed_plugins.append(
scalars_metadata.PLUGIN_NAME)
actual = server_info.allowed_plugins(info)
self.assertEqual(actual, frozenset([scalars_metadata.PLUGIN_NAME]))
def test_provided_but_no_plugins(self):
info = server_info_pb2.ServerInfoResponse()
info.plugin_control.SetInParent()
actual = server_info.allowed_plugins(info)
self.assertEqual(actual, frozenset([]))
def test_old_server_no_plugins(self):
info = server_info_pb2.ServerInfoResponse()
actual = server_info.allowed_plugins(info)
self.assertEqual(actual, frozenset([scalars_metadata.PLUGIN_NAME]))
def test(self):
info = server_info_pb2.ServerInfoResponse()
info.url_format.template = "https://unittest.tensorboard.dev/x/???"
info.url_format.id_placeholder = "???"
actual = server_info.experiment_url(info, "123")
self.assertEqual(actual, "https://unittest.tensorboard.dev/x/123")
def app(request):
result = server_info_pb2.ServerInfoResponse()
result.compatibility.details = request.headers["User-Agent"]
return wrappers.BaseResponse(result.SerializeToString())
def test_old_server_no_upload_limits(self):
info = server_info_pb2.ServerInfoResponse()
actual = server_info.max_blob_size(info)
self.assertEqual(actual, server_info._DEFAULT_MAX_BLOB_SIZE)
def test_upload_limits_provided_with_max_blob_size(self):
info = server_info_pb2.ServerInfoResponse()
info.upload_limits.max_blob_size = 42
actual = server_info.max_blob_size(info)
self.assertEqual(actual, 42)