def test_call_with_retries_fails_immediately_on_permanent_error(self):
def handler(_, context):
context.abort(grpc.StatusCode.INTERNAL, "foo")
server = TestGrpcServer(handler)
with server.run() as client:
with self.assertRaises(grpc.RpcError) as raised:
grpc_util.call_with_retries(client.TestRpc, make_request(42))
self.assertEqual(grpc.StatusCode.INTERNAL, raised.exception.code())
self.assertEqual("foo", raised.exception.details())
def test_call_with_retries_succeeds(self):
def handler(request, _):
return make_response(request.nonce)
server = TestGrpcServer(handler)
with server.run() as client:
response = grpc_util.call_with_retries(client.TestRpc, make_request(42))
self.assertEqual(make_response(42), response)
def _get_or_create_blob_sequence(self):
request = write_service_pb2.GetOrCreateBlobSequenceRequest(
experiment_id=self._experiment_id,
run=self._run_name,
tag=self._value.tag,
step=self._event.step,
final_sequence_length=len(self._blobs),
metadata=self._metadata,
)
util.set_timestamp(request.wall_time, self._event.wall_time)
with _request_logger(request):
try:
# TODO(@nfelt): execute this RPC asynchronously.
response = grpc_util.call_with_retries(
self._api.GetOrCreateBlobSequence, request
)
blob_sequence_id = response.blob_sequence_id
except grpc.RpcError as e:
if e.code() == grpc.StatusCode.NOT_FOUND:
raise ExperimentNotFoundError()
logger.error("Upload call failed with error %s", e)
# TODO(soergel): clean up
raise
return blob_sequence_id
def create_experiment(self):
"""Creates an Experiment for this upload session and returns the ID."""
logger.info("Creating experiment")
request = write_service_pb2.CreateExperimentRequest()
response = grpc_util.call_with_retries(self._api.CreateExperiment,
request)
self._request_builder = _RequestBuilder(response.experiment_id)
return response.experiment_id
def test_call_with_retries_fails_after_backoff_on_nonpermanent_error(self):
attempt_times = []
fake_time = test_util.FakeTime()
def handler(_, context):
attempt_times.append(fake_time.time())
context.abort(grpc.StatusCode.UNAVAILABLE, "foo")
server = TestGrpcServer(handler)
with server.run() as client:
with self.assertRaises(grpc.RpcError) as raised:
grpc_util.call_with_retries(client.TestRpc, make_request(42), fake_time)
self.assertEqual(grpc.StatusCode.UNAVAILABLE, raised.exception.code())
self.assertEqual("foo", raised.exception.details())
self.assertLen(attempt_times, 5)
self.assertBetween(attempt_times[1] - attempt_times[0], 2, 4)
self.assertBetween(attempt_times[2] - attempt_times[1], 4, 8)
self.assertBetween(attempt_times[3] - attempt_times[2], 8, 16)
self.assertBetween(attempt_times[4] - attempt_times[3], 16, 32)
def update_experiment_metadata(writer_client,
experiment_id,
name=None,
description=None):
"""Modifies user data associated with an experiment.
Args:
writer_client: a TensorBoardWriterService stub instance
experiment_id: string ID of the experiment to modify
name: If provided, modifies name of experiment to this value.
description: If provided, modifies the description of the experiment to
this value
Raises:
ExperimentNotFoundError: If no such experiment exists.
PermissionDeniedError: If the user is not authorized to modify this
experiment.
InvalidArgumentError: If the server rejected the name or description, if,
for instance, the size limits have changed on the server.
"""
logger.info("Modifying experiment %r", experiment_id)
request = write_service_pb2.UpdateExperimentRequest()
request.experiment.experiment_id = experiment_id
if name is not None:
logger.info("Setting exp %r name to %r", experiment_id, name)
request.experiment.name = name
request.experiment_mask.name = True
if description is not None:
logger.info("Setting exp %r description to %r", experiment_id,
description)
request.experiment.description = description
request.experiment_mask.description = True
try:
grpc_util.call_with_retries(writer_client.UpdateExperiment, request)
except grpc.RpcError as e:
if e.code() == grpc.StatusCode.NOT_FOUND:
raise ExperimentNotFoundError()
if e.code() == grpc.StatusCode.PERMISSION_DENIED:
raise PermissionDeniedError()
if e.code() == grpc.StatusCode.INVALID_ARGUMENT:
raise InvalidArgumentError(e.details())
raise
def create_experiment(self):
"""Creates an Experiment for this upload session and returns the ID."""
logger.info("Creating experiment")
request = write_service_pb2.CreateExperimentRequest(
name=self._name, description=self._description)
response = grpc_util.call_with_retries(self._api.CreateExperiment,
request)
self._request_sender = _BatchedRequestSender(response.experiment_id,
self._api,
self._rpc_rate_limiter)
return response.experiment_id
def flush(self):
"""Sends the active request after removing empty runs and tags.
Starts a new, empty active request.
"""
request = self._request
_prune_empty_tags_and_runs(request)
if not request.runs:
return
self._rpc_rate_limiter.tick()
with _request_logger(request, request.runs):
try:
grpc_util.call_with_retries(self._api.WriteTensor, request)
except grpc.RpcError as e:
if e.code() == grpc.StatusCode.NOT_FOUND:
raise ExperimentNotFoundError()
logger.error("Upload call failed with error %s", e)
self._new_request()
def flush(self):
"""Sends the active request after removing empty runs and tags.
Starts a new, empty active request.
"""
request = self._request
_prune_empty_tags_and_runs(request)
if not request.runs:
return
self._rpc_rate_limiter.tick()
with _request_logger(request, request.runs):
with self._tracker.scalars_tracker(self._num_values):
try:
# TODO(@nfelt): execute this RPC asynchronously.
grpc_util.call_with_retries(self._api.WriteScalar, request)
except grpc.RpcError as e:
if e.code() == grpc.StatusCode.NOT_FOUND:
raise ExperimentNotFoundError()
logger.error("Upload call failed with error %s", e)
self._new_request()
def test_call_with_retries_includes_version_metadata(self):
def digest(s):
"""Hashes a string into a 32-bit integer."""
return int(hashlib.sha256(s.encode("utf-8")).hexdigest(), 16) & 0xffffffff
def handler(request, context):
metadata = context.invocation_metadata()
client_version = grpc_util.extract_version(metadata)
return make_response(digest(client_version))
server = TestGrpcServer(handler)
with server.run() as client:
response = grpc_util.call_with_retries(client.TestRpc, make_request(0))
expected_nonce = digest(
grpc_util.extract_version(grpc_util.version_metadata()))
self.assertEqual(make_response(expected_nonce), response)
def delete_experiment(writer_client, experiment_id):
"""Permanently deletes an experiment and all of its contents.
Args:
writer_client: a TensorBoardWriterService stub instance
experiment_id: string ID of the experiment to delete
Raises:
ExperimentNotFoundError: If no such experiment exists.
PermissionDeniedError: If the user is not authorized to delete this
experiment.
RuntimeError: On unexpected failure.
"""
logger.info("Deleting experiment %r", experiment_id)
request = write_service_pb2.DeleteExperimentRequest()
request.experiment_id = experiment_id
try:
grpc_util.call_with_retries(writer_client.DeleteExperiment, request)
except grpc.RpcError as e:
if e.code() == grpc.StatusCode.NOT_FOUND:
raise ExperimentNotFoundError()
if e.code() == grpc.StatusCode.PERMISSION_DENIED:
raise PermissionDeniedError()
raise
def test_call_with_retries_succeeds_after_backoff_on_transient_error(self):
attempt_times = []
fake_time = test_util.FakeTime()
def handler(request, context):
attempt_times.append(fake_time.time())
if len(attempt_times) < 3:
context.abort(grpc.StatusCode.UNAVAILABLE, "foo")
return make_response(request.nonce)
server = TestGrpcServer(handler)
with server.run() as client:
response = grpc_util.call_with_retries(
client.TestRpc, make_request(42), fake_time)
self.assertEqual(make_response(42), response)
self.assertLen(attempt_times, 3)
self.assertBetween(attempt_times[1] - attempt_times[0], 2, 4)
self.assertBetween(attempt_times[2] - attempt_times[1], 4, 8)
def create_experiment(self):
"""Creates an Experiment for this upload session and returns the ID."""
logger.info("Creating experiment")
request = write_service_pb2.CreateExperimentRequest(
name=self._name, description=self._description)
response = grpc_util.call_with_retries(self._api.CreateExperiment,
request)
self._tracker = upload_tracker.UploadTracker(verbosity=self._verbosity)
self._request_sender = _BatchedRequestSender(
response.experiment_id,
self._api,
allowed_plugins=self._allowed_plugins,
upload_limits=self._upload_limits,
rpc_rate_limiter=self._rpc_rate_limiter,
tensor_rpc_rate_limiter=self._tensor_rpc_rate_limiter,
blob_rpc_rate_limiter=self._blob_rpc_rate_limiter,
tracker=self._tracker,
)
return response.experiment_id
