def test_call_with_retries_fails_immediately_on_permanent_error(self): def handler(_, context): context.abort(grpc.StatusCode.INTERNAL, "foo") server = TestGrpcServer(handler) with server.run() as client: with self.assertRaises(grpc.RpcError) as raised: grpc_util.call_with_retries(client.TestRpc, make_request(42)) self.assertEqual(grpc.StatusCode.INTERNAL, raised.exception.code()) self.assertEqual("foo", raised.exception.details())
def test_call_with_retries_succeeds(self): def handler(request, _): return make_response(request.nonce) server = TestGrpcServer(handler) with server.run() as client: response = grpc_util.call_with_retries(client.TestRpc, make_request(42)) self.assertEqual(make_response(42), response)
def _get_or_create_blob_sequence(self): request = write_service_pb2.GetOrCreateBlobSequenceRequest( experiment_id=self._experiment_id, run=self._run_name, tag=self._value.tag, step=self._event.step, final_sequence_length=len(self._blobs), metadata=self._metadata, ) util.set_timestamp(request.wall_time, self._event.wall_time) with _request_logger(request): try: # TODO(@nfelt): execute this RPC asynchronously. response = grpc_util.call_with_retries( self._api.GetOrCreateBlobSequence, request ) blob_sequence_id = response.blob_sequence_id except grpc.RpcError as e: if e.code() == grpc.StatusCode.NOT_FOUND: raise ExperimentNotFoundError() logger.error("Upload call failed with error %s", e) # TODO(soergel): clean up raise return blob_sequence_id
def create_experiment(self): """Creates an Experiment for this upload session and returns the ID.""" logger.info("Creating experiment") request = write_service_pb2.CreateExperimentRequest() response = grpc_util.call_with_retries(self._api.CreateExperiment, request) self._request_builder = _RequestBuilder(response.experiment_id) return response.experiment_id
def test_call_with_retries_fails_after_backoff_on_nonpermanent_error(self): attempt_times = [] fake_time = test_util.FakeTime() def handler(_, context): attempt_times.append(fake_time.time()) context.abort(grpc.StatusCode.UNAVAILABLE, "foo") server = TestGrpcServer(handler) with server.run() as client: with self.assertRaises(grpc.RpcError) as raised: grpc_util.call_with_retries(client.TestRpc, make_request(42), fake_time) self.assertEqual(grpc.StatusCode.UNAVAILABLE, raised.exception.code()) self.assertEqual("foo", raised.exception.details()) self.assertLen(attempt_times, 5) self.assertBetween(attempt_times[1] - attempt_times[0], 2, 4) self.assertBetween(attempt_times[2] - attempt_times[1], 4, 8) self.assertBetween(attempt_times[3] - attempt_times[2], 8, 16) self.assertBetween(attempt_times[4] - attempt_times[3], 16, 32)
def update_experiment_metadata(writer_client, experiment_id, name=None, description=None): """Modifies user data associated with an experiment. Args: writer_client: a TensorBoardWriterService stub instance experiment_id: string ID of the experiment to modify name: If provided, modifies name of experiment to this value. description: If provided, modifies the description of the experiment to this value Raises: ExperimentNotFoundError: If no such experiment exists. PermissionDeniedError: If the user is not authorized to modify this experiment. InvalidArgumentError: If the server rejected the name or description, if, for instance, the size limits have changed on the server. """ logger.info("Modifying experiment %r", experiment_id) request = write_service_pb2.UpdateExperimentRequest() request.experiment.experiment_id = experiment_id if name is not None: logger.info("Setting exp %r name to %r", experiment_id, name) request.experiment.name = name request.experiment_mask.name = True if description is not None: logger.info("Setting exp %r description to %r", experiment_id, description) request.experiment.description = description request.experiment_mask.description = True try: grpc_util.call_with_retries(writer_client.UpdateExperiment, request) except grpc.RpcError as e: if e.code() == grpc.StatusCode.NOT_FOUND: raise ExperimentNotFoundError() if e.code() == grpc.StatusCode.PERMISSION_DENIED: raise PermissionDeniedError() if e.code() == grpc.StatusCode.INVALID_ARGUMENT: raise InvalidArgumentError(e.details()) raise
def create_experiment(self): """Creates an Experiment for this upload session and returns the ID.""" logger.info("Creating experiment") request = write_service_pb2.CreateExperimentRequest( name=self._name, description=self._description) response = grpc_util.call_with_retries(self._api.CreateExperiment, request) self._request_sender = _BatchedRequestSender(response.experiment_id, self._api, self._rpc_rate_limiter) return response.experiment_id
def flush(self): """Sends the active request after removing empty runs and tags. Starts a new, empty active request. """ request = self._request _prune_empty_tags_and_runs(request) if not request.runs: return self._rpc_rate_limiter.tick() with _request_logger(request, request.runs): try: grpc_util.call_with_retries(self._api.WriteTensor, request) except grpc.RpcError as e: if e.code() == grpc.StatusCode.NOT_FOUND: raise ExperimentNotFoundError() logger.error("Upload call failed with error %s", e) self._new_request()
def flush(self): """Sends the active request after removing empty runs and tags. Starts a new, empty active request. """ request = self._request _prune_empty_tags_and_runs(request) if not request.runs: return self._rpc_rate_limiter.tick() with _request_logger(request, request.runs): with self._tracker.scalars_tracker(self._num_values): try: # TODO(@nfelt): execute this RPC asynchronously. grpc_util.call_with_retries(self._api.WriteScalar, request) except grpc.RpcError as e: if e.code() == grpc.StatusCode.NOT_FOUND: raise ExperimentNotFoundError() logger.error("Upload call failed with error %s", e) self._new_request()
def test_call_with_retries_includes_version_metadata(self): def digest(s): """Hashes a string into a 32-bit integer.""" return int(hashlib.sha256(s.encode("utf-8")).hexdigest(), 16) & 0xffffffff def handler(request, context): metadata = context.invocation_metadata() client_version = grpc_util.extract_version(metadata) return make_response(digest(client_version)) server = TestGrpcServer(handler) with server.run() as client: response = grpc_util.call_with_retries(client.TestRpc, make_request(0)) expected_nonce = digest( grpc_util.extract_version(grpc_util.version_metadata())) self.assertEqual(make_response(expected_nonce), response)
def delete_experiment(writer_client, experiment_id): """Permanently deletes an experiment and all of its contents. Args: writer_client: a TensorBoardWriterService stub instance experiment_id: string ID of the experiment to delete Raises: ExperimentNotFoundError: If no such experiment exists. PermissionDeniedError: If the user is not authorized to delete this experiment. RuntimeError: On unexpected failure. """ logger.info("Deleting experiment %r", experiment_id) request = write_service_pb2.DeleteExperimentRequest() request.experiment_id = experiment_id try: grpc_util.call_with_retries(writer_client.DeleteExperiment, request) except grpc.RpcError as e: if e.code() == grpc.StatusCode.NOT_FOUND: raise ExperimentNotFoundError() if e.code() == grpc.StatusCode.PERMISSION_DENIED: raise PermissionDeniedError() raise
def test_call_with_retries_succeeds_after_backoff_on_transient_error(self): attempt_times = [] fake_time = test_util.FakeTime() def handler(request, context): attempt_times.append(fake_time.time()) if len(attempt_times) < 3: context.abort(grpc.StatusCode.UNAVAILABLE, "foo") return make_response(request.nonce) server = TestGrpcServer(handler) with server.run() as client: response = grpc_util.call_with_retries( client.TestRpc, make_request(42), fake_time) self.assertEqual(make_response(42), response) self.assertLen(attempt_times, 3) self.assertBetween(attempt_times[1] - attempt_times[0], 2, 4) self.assertBetween(attempt_times[2] - attempt_times[1], 4, 8)
def create_experiment(self): """Creates an Experiment for this upload session and returns the ID.""" logger.info("Creating experiment") request = write_service_pb2.CreateExperimentRequest( name=self._name, description=self._description) response = grpc_util.call_with_retries(self._api.CreateExperiment, request) self._tracker = upload_tracker.UploadTracker(verbosity=self._verbosity) self._request_sender = _BatchedRequestSender( response.experiment_id, self._api, allowed_plugins=self._allowed_plugins, upload_limits=self._upload_limits, rpc_rate_limiter=self._rpc_rate_limiter, tensor_rpc_rate_limiter=self._tensor_rpc_rate_limiter, blob_rpc_rate_limiter=self._blob_rpc_rate_limiter, tracker=self._tracker, ) return response.experiment_id