def test_globaldb_endpoint_discovery(self): connection_policy = documents.ConnectionPolicy() connection_policy.EnableEndpointDiscovery = False read_location_client = cosmos_client_connection.CosmosClientConnection(Test_globaldb_tests.read_location_host, Test_globaldb_tests.masterKey, connection_policy) document_definition = { 'id': 'doc', 'name': 'sample document', 'key': 'value'} # Create Document will fail for the read location client since it has EnableEndpointDiscovery set to false, and hence the request will directly go to # the endpoint that was used to create the client instance(which happens to be a read endpoint) self.__AssertHTTPFailureWithStatus( StatusCodes.FORBIDDEN, SubStatusCodes.WRITE_FORBIDDEN, read_location_client.CreateItem, self.test_coll['_self'], document_definition) # Query databases will pass for the read location client as it's a GET operation list(read_location_client.QueryDatabases({ 'query': 'SELECT * FROM root r WHERE r.id=@id', 'parameters': [ { 'name':'@id', 'value': self.test_db['id'] } ] })) connection_policy.EnableEndpointDiscovery = True read_location_client = cosmos_client_connection.CosmosClientConnection(Test_globaldb_tests.read_location_host, Test_globaldb_tests.masterKey, connection_policy) # CreateDocument call will go to the WriteEndpoint as EnableEndpointDiscovery is set to True and client will resolve the right endpoint based on the operation created_document = read_location_client.CreateItem(self.test_coll['_self'], document_definition) self.assertEqual(created_document['id'], document_definition['id'])
def test_globaldb_preferred_locations(self): connection_policy = documents.ConnectionPolicy() connection_policy.EnableEndpointDiscovery = True client = cosmos_client_connection.CosmosClientConnection( Test_globaldb_tests.host, {'masterKey': Test_globaldb_tests.masterKey}, connection_policy) document_definition = { 'id': 'doc', 'name': 'sample document', 'key': 'value' } created_document = client.CreateItem(self.test_coll['_self'], document_definition) self.assertEqual(created_document['id'], document_definition['id']) # Delay to get these resources replicated to read location due to Eventual consistency time.sleep(5) client.ReadItem(created_document['_self']) content_location = str( client.last_response_headers[HttpHeaders.ContentLocation]) content_location_url = urlparse(content_location) write_location_url = urlparse(Test_globaldb_tests.write_location_host) # If no preferred locations is set, we return the write endpoint as ReadEndpoint for better latency performance self.assertEqual(str(content_location_url.hostname), str(write_location_url.hostname)) self.assertEqual(client.ReadEndpoint, Test_globaldb_tests.write_location_host) connection_policy.PreferredLocations = [ Test_globaldb_tests.read_location2 ] client = cosmos_client_connection.CosmosClientConnection( Test_globaldb_tests.host, {'masterKey': Test_globaldb_tests.masterKey}, connection_policy) document_definition['id'] = 'doc2' created_document = client.CreateItem(self.test_coll['_self'], document_definition) # Delay to get these resources replicated to read location due to Eventual consistency time.sleep(5) client.ReadItem(created_document['_self']) content_location = str( client.last_response_headers[HttpHeaders.ContentLocation]) content_location_url = urlparse(content_location) read_location2_url = urlparse(Test_globaldb_tests.read_location2_host) # Test that the preferred location is set as ReadEndpoint instead of default write endpoint when no preference is set self.assertEqual(str(content_location_url.hostname), str(read_location2_url.hostname)) self.assertEqual(client.ReadEndpoint, Test_globaldb_tests.read_location2_host)
def test_globaldb_read_write_endpoints(self): connection_policy = documents.ConnectionPolicy() connection_policy.EnableEndpointDiscovery = False client = cosmos_client_connection.CosmosClientConnection(Test_globaldb_tests.host, Test_globaldb_tests.masterKey, connection_policy) document_definition = { 'id': 'doc', 'name': 'sample document', 'key': 'value'} # When EnableEndpointDiscovery is False, WriteEndpoint is set to the endpoint passed while creating the client instance created_document = client.CreateItem(self.test_coll['_self'], document_definition) self.assertEqual(client.WriteEndpoint, Test_globaldb_tests.host) # Delay to get these resources replicated to read location due to Eventual consistency time.sleep(5) client.ReadItem(created_document['_self']) content_location = str(client.last_response_headers[HttpHeaders.ContentLocation]) content_location_url = urlparse(content_location) host_url = urlparse(Test_globaldb_tests.host) # When EnableEndpointDiscovery is False, ReadEndpoint is set to the endpoint passed while creating the client instance self.assertEqual(str(content_location_url.hostname), str(host_url.hostname)) self.assertEqual(client.ReadEndpoint, Test_globaldb_tests.host) connection_policy.EnableEndpointDiscovery = True document_definition['id'] = 'doc2' client = cosmos_client_connection.CosmosClientConnection(Test_globaldb_tests.host, Test_globaldb_tests.masterKey, connection_policy) # When EnableEndpointDiscovery is True, WriteEndpoint is set to the write endpoint created_document = client.CreateItem(self.test_coll['_self'], document_definition) self.assertEqual(client.WriteEndpoint, Test_globaldb_tests.write_location_host) # Delay to get these resources replicated to read location due to Eventual consistency time.sleep(5) client.ReadItem(created_document['_self']) content_location = str(client.last_response_headers[HttpHeaders.ContentLocation]) content_location_url = urlparse(content_location) write_location_url = urlparse(Test_globaldb_tests.write_location_host) # If no preferred locations is set, we return the write endpoint as ReadEndpoint for better latency performance self.assertEqual(str(content_location_url.hostname), str(write_location_url.hostname)) self.assertEqual(client.ReadEndpoint, Test_globaldb_tests.write_location_host)
def test_globaldb_endpoint_discovery_retry_policy_mock(self): client = cosmos_client_connection.CosmosClientConnection( Test_globaldb_tests.host, {'masterKey': Test_globaldb_tests.masterKey}) self.OriginalExecuteFunction = _retry_utility.ExecuteFunction _retry_utility.ExecuteFunction = self._MockExecuteFunction self.OriginalGetDatabaseAccount = client.GetDatabaseAccount client.GetDatabaseAccount = self._MockGetDatabaseAccount max_retry_attempt_count = 10 retry_after_in_milliseconds = 500 _endpoint_discovery_retry_policy.EndpointDiscoveryRetryPolicy.Max_retry_attempt_count = max_retry_attempt_count _endpoint_discovery_retry_policy.EndpointDiscoveryRetryPolicy.Retry_after_in_milliseconds = retry_after_in_milliseconds document_definition = { 'id': 'doc', 'name': 'sample document', 'key': 'value' } self.__AssertHTTPFailureWithStatus(StatusCodes.FORBIDDEN, SubStatusCodes.WRITE_FORBIDDEN, client.CreateItem, self.test_coll['_self'], document_definition) _retry_utility.ExecuteFunction = self.OriginalExecuteFunction
def __init__(self): self.account_endpoint = Configurations.ENDPOINT self.account_key = Configurations.ACCOUNT_KEY self.regions = Configurations.REGIONS.split(';') self.database_name = Configurations.DATABASE_NAME self.manual_collection_name = Configurations.MANUAL_COLLECTION_NAME self.lww_collection_name = Configurations.LWW_COLLECTION_NAME self.udp_collection_name = Configurations.UDP_COLLECTION_NAME self.basic_collection_name = Configurations.BASIC_COLLECTION_NAME self.workers = [] self.conflict_worker = ConflictWorker(self.database_name, self.basic_collection_name, self.manual_collection_name, self.lww_collection_name, self.udp_collection_name) self.pool = ThreadPool(processes=len(self.regions)) for region in self.regions: connection_policy = documents.ConnectionPolicy() connection_policy.UseMultipleWriteLocations = True connection_policy.PreferredLocations = [region] client = cosmos_client_connection.CosmosClientConnection( self.account_endpoint, {'masterKey': self.account_key}, connection_policy, documents.ConsistencyLevel.Session) self.workers.append( Worker(client, self.database_name, self.basic_collection_name)) self.conflict_worker.add_client(client)
def setUp(self): self.client = cosmos_client_connection.CosmosClientConnection( Test_globaldb_tests.host, {'masterKey': Test_globaldb_tests.masterKey}) # Create the test database only when it's not already present query_iterable = self.client.QueryDatabases( 'SELECT * FROM root r WHERE r.id=\'' + Test_globaldb_tests.test_database_id + '\'') it = iter(query_iterable) self.test_db = next(it, None) if self.test_db is None: self.test_db = self.client.CreateDatabase( {'id': Test_globaldb_tests.test_database_id}) # Create the test collection only when it's not already present query_iterable = self.client.QueryContainers( self.test_db['_self'], 'SELECT * FROM root r WHERE r.id=\'' + Test_globaldb_tests.test_collection_id + '\'') it = iter(query_iterable) self.test_coll = next(it, None) if self.test_coll is None: self.test_coll = self.client.CreateContainer( self.test_db['_self'], {'id': Test_globaldb_tests.test_collection_id})
def test_success_with_correct_proxy(self): connection_policy.ProxyConfiguration.Port = self.serverPort client = cosmos_client_connection.CosmosClientConnection( self.host, {'masterKey': self.masterKey}, connection_policy) created_db = client.CreateDatabase({'id': self.testDbName}) self.assertEqual(created_db['id'], self.testDbName, msg="Database id is incorrect")
def test_failure_with_wrong_proxy(self): connection_policy.ProxyConfiguration.Port = self.serverPort + 1 try: # client does a getDatabaseAccount on initialization, which fails client = cosmos_client_connection.CosmosClientConnection(self.host, {'masterKey': self.masterKey}, connection_policy) self.fail("Client instantiation is not expected") except Exception as e: self.assertTrue(type(e) is ServiceRequestError, msg="Error is not a ServiceRequestError")
def test_globaldb_endpoint_discovery_retry_policy(self): connection_policy = documents.ConnectionPolicy() connection_policy.EnableEndpointDiscovery = True write_location_client = cosmos_client_connection.CosmosClientConnection(Test_globaldb_mock_tests.write_location_host, Test_globaldb_mock_tests.masterKey, connection_policy) self.assertEqual(write_location_client._global_endpoint_manager.WriteEndpoint, Test_globaldb_mock_tests.write_location_host) self.MockCreateDatabase(write_location_client, { 'id': 'mock database' }) self.assertEqual(write_location_client._global_endpoint_manager.WriteEndpoint, Test_globaldb_mock_tests.read_location_host)
def create_spy_client(self, use_multiple_write_locations, enable_endpoint_discovery, is_preferred_locations_list_empty): self.preferred_locations = ["location1", "location2", "location3", "location4"] connectionPolicy = documents.ConnectionPolicy() connectionPolicy.DisableSSLVerification = True connectionPolicy.PreferredLocations = [] if is_preferred_locations_list_empty else self.preferred_locations connectionPolicy.EnableEndpointDiscovery = enable_endpoint_discovery connectionPolicy.UseMultipleWriteLocations = use_multiple_write_locations client = cosmos_client_connection.CosmosClientConnection(self.DEFAULT_ENDPOINT, {'masterKey': "SomeKeyValue"}, connection_policy=connectionPolicy) return client
def test_success_with_correct_proxy(self): if platform.system() == 'Darwin': pytest.skip("TODO: Connection error raised on OSX") connection_policy.ProxyConfiguration.Port = self.serverPort client = cosmos_client_connection.CosmosClientConnection( self.host, {'masterKey': self.masterKey}, connection_policy) created_db = client.CreateDatabase({'id': self.testDbName}) self.assertEqual(created_db['id'], self.testDbName, msg="Database id is incorrect")
def test_globaldb_endpoint_assignments(self): connection_policy = documents.ConnectionPolicy() connection_policy.EnableEndpointDiscovery = False client = cosmos_client_connection.CosmosClientConnection(Test_globaldb_tests.host, Test_globaldb_tests.masterKey, connection_policy) # When EnableEndpointDiscovery is set to False, both Read and Write Endpoints point to endpoint passed while creating the client instance self.assertEqual(client._global_endpoint_manager.WriteEndpoint, Test_globaldb_tests.host) self.assertEqual(client._global_endpoint_manager.ReadEndpoint, Test_globaldb_tests.host) connection_policy.EnableEndpointDiscovery = True client = cosmos_client_connection.CosmosClientConnection(Test_globaldb_tests.host, Test_globaldb_tests.masterKey, connection_policy) # If no preferred locations is set, we return the write endpoint as ReadEndpoint for better latency performance, write endpoint is set as expected self.assertEqual(client._global_endpoint_manager.WriteEndpoint, Test_globaldb_tests.write_location_host) self.assertEqual(client._global_endpoint_manager.ReadEndpoint, Test_globaldb_tests.write_location_host) connection_policy.PreferredLocations = [Test_globaldb_tests.read_location2] client = cosmos_client_connection.CosmosClientConnection(Test_globaldb_tests.host, Test_globaldb_tests.masterKey, connection_policy) # Test that the preferred location is set as ReadEndpoint instead of default write endpoint when no preference is set self.assertEqual(client._global_endpoint_manager.WriteEndpoint, Test_globaldb_tests.write_location_host) self.assertEqual(client._global_endpoint_manager.ReadEndpoint, Test_globaldb_tests.read_location2_host)
def initialize(self, use_multiple_write_locations, enable_endpoint_discovery, is_preferred_locations_list_empty): self.database_account = self.create_database_account(use_multiple_write_locations) preferred_locations = ["location1", "location2", "location3"] self.preferred_locations = [] if is_preferred_locations_list_empty else preferred_locations self.location_cache = LocationCache( self.preferred_locations, self.DEFAULT_ENDPOINT, enable_endpoint_discovery, use_multiple_write_locations, self.REFRESH_TIME_INTERVAL_IN_MS) self.location_cache.perform_on_database_account_read(self.database_account) connectionPolicy = documents.ConnectionPolicy() connectionPolicy.PreferredLocations = self.preferred_locations client = cosmos_client_connection.CosmosClientConnection("", {}, connection_policy=connectionPolicy) self.global_endpoint_manager = client._global_endpoint_manager
def test_retry_policy_does_not_mark_null_locations_unavailable(self): self.original_get_database_account = cosmos_client_connection.CosmosClientConnection.GetDatabaseAccount cosmos_client_connection.CosmosClientConnection.GetDatabaseAccount = self.mock_get_database_account client = cosmos_client_connection.CosmosClientConnection( self.DEFAULT_ENDPOINT, {'masterKey': self.MASTER_KEY}, None, documents.ConsistencyLevel.Eventual) endpoint_manager = global_endpoint_manager._GlobalEndpointManager( client) self.original_mark_endpoint_unavailable_for_read_function = endpoint_manager.mark_endpoint_unavailable_for_read endpoint_manager.mark_endpoint_unavailable_for_read = self._mock_mark_endpoint_unavailable_for_read self.original_mark_endpoint_unavailable_for_write_function = endpoint_manager.mark_endpoint_unavailable_for_write endpoint_manager.mark_endpoint_unavailable_for_write = self._mock_mark_endpoint_unavailable_for_write self.original_resolve_service_endpoint = endpoint_manager.resolve_service_endpoint endpoint_manager.resolve_service_endpoint = self._mock_resolve_service_endpoint # Read and write counters count the number of times the endpoint manager's # mark_endpoint_unavailable_for_read() and mark_endpoint_unavailable_for_read() # functions were called. When a 'None' location is returned by resolve_service_endpoint(), # these functions should not be called self._read_counter = 0 self._write_counter = 0 request = RequestObject(http_constants.ResourceType.Document, documents._OperationType.Read) endpointDiscovery_retry_policy = _endpoint_discovery_retry_policy.EndpointDiscoveryRetryPolicy( documents.ConnectionPolicy(), endpoint_manager, request) endpointDiscovery_retry_policy.ShouldRetry( errors.CosmosHttpResponseError( status_code=http_constants.StatusCodes.FORBIDDEN)) self.assertEqual(self._read_counter, 0) self.assertEqual(self._write_counter, 0) self._read_counter = 0 self._write_counter = 0 request = RequestObject(http_constants.ResourceType.Document, documents._OperationType.Create) endpointDiscovery_retry_policy = _endpoint_discovery_retry_policy.EndpointDiscoveryRetryPolicy( documents.ConnectionPolicy(), endpoint_manager, request) endpointDiscovery_retry_policy.ShouldRetry( errors.CosmosHttpResponseError( status_code=http_constants.StatusCodes.FORBIDDEN)) self.assertEqual(self._read_counter, 0) self.assertEqual(self._write_counter, 0) endpoint_manager.mark_endpoint_unavailable_for_read = self.original_mark_endpoint_unavailable_for_read_function endpoint_manager.mark_endpoint_unavailable_for_write = self.original_mark_endpoint_unavailable_for_write_function cosmos_client_connection.CosmosClientConnection.GetDatabaseAccount = self.original_get_database_account
def test_globaldb_database_account_unavailable(self): connection_policy = documents.ConnectionPolicy() connection_policy.EnableEndpointDiscovery = True client = cosmos_client_connection.CosmosClientConnection(Test_globaldb_mock_tests.host, Test_globaldb_mock_tests.masterKey, connection_policy) self.assertEqual(client._global_endpoint_manager.WriteEndpoint, Test_globaldb_mock_tests.write_location_host) self.assertEqual(client._global_endpoint_manager.ReadEndpoint, Test_globaldb_mock_tests.write_location_host) global_endpoint_manager._GlobalEndpointManager._GetDatabaseAccountStub = self.MockGetDatabaseAccountStub client._global_endpoint_manager.DatabaseAccountAvailable = False client._global_endpoint_manager.RefreshEndpointList() self.assertEqual(client._global_endpoint_manager.WriteEndpoint, Test_globaldb_mock_tests.host) self.assertEqual(client._global_endpoint_manager.ReadEndpoint, Test_globaldb_mock_tests.host)
def test_streaming_failover(self): self.OriginalExecuteFunction = _retry_utility.ExecuteFunction _retry_utility.ExecuteFunction = self._MockExecuteFunctionEndpointDiscover connection_policy = documents.ConnectionPolicy() connection_policy.PreferredLocations = self.preferred_regional_endpoints connection_policy.DisableSSLVerification = True self.original_get_database_account = cosmos_client_connection.CosmosClientConnection.GetDatabaseAccount cosmos_client_connection.CosmosClientConnection.GetDatabaseAccount = self.mock_get_database_account client = cosmos_client_connection.CosmosClientConnection( self.DEFAULT_ENDPOINT, {'masterKey': self.MASTER_KEY}, connection_policy, documents.ConsistencyLevel.Eventual) document_definition = { 'id': 'doc', 'name': 'sample document', 'key': 'value' } created_document = {} created_document = client.CreateItem("dbs/mydb/colls/mycoll", document_definition) self.assertDictEqual(created_document, {}) self.assertDictEqual(client.last_response_headers, {}) self.assertEqual(self.counter, 10) # First request is an initial read collection. # Next 8 requests hit forbidden write exceptions and the endpoint retry policy keeps # flipping the resolved endpoint between the 2 write endpoints. # The 10th request returns the actual read document. for i in range(0, 8): if i % 2 == 0: self.assertEqual(self.endpoint_sequence[i], self.WRITE_ENDPOINT1) else: self.assertEqual(self.endpoint_sequence[i], self.WRITE_ENDPOINT2) cosmos_client_connection.CosmosClientConnection.GetDatabaseAccount = self.original_get_database_account _retry_utility.ExecuteFunction = self.OriginalExecuteFunction
def test_globaldb_update_locations_cache(self): client = cosmos_client_connection.CosmosClientConnection( Test_globaldb_tests.host, {'masterKey': Test_globaldb_tests.masterKey}) writable_locations = [{ 'name': Test_globaldb_tests.write_location, 'databaseAccountEndpoint': Test_globaldb_tests.write_location_host }] readable_locations = [{ 'name': Test_globaldb_tests.read_location, 'databaseAccountEndpoint': Test_globaldb_tests.read_location_host }, { 'name': Test_globaldb_tests.read_location2, 'databaseAccountEndpoint': Test_globaldb_tests.read_location2_host }] write_endpoint, read_endpoint = client._global_endpoint_manager.UpdateLocationsCache( writable_locations, readable_locations) # If no preferred locations is set, we return the write endpoint as ReadEndpoint for better latency performance, write endpoint is set as expected self.assertEqual(write_endpoint, Test_globaldb_tests.write_location_host) self.assertEqual(read_endpoint, Test_globaldb_tests.write_location_host) writable_locations = [] readable_locations = [] write_endpoint, read_endpoint = client._global_endpoint_manager.UpdateLocationsCache( writable_locations, readable_locations) # If writable_locations and readable_locations are empty, both Read and Write Endpoints point to endpoint passed while creating the client instance self.assertEqual(write_endpoint, Test_globaldb_tests.host) self.assertEqual(read_endpoint, Test_globaldb_tests.host) writable_locations = [{ 'name': Test_globaldb_tests.write_location, 'databaseAccountEndpoint': Test_globaldb_tests.write_location_host }] readable_locations = [] write_endpoint, read_endpoint = client._global_endpoint_manager.UpdateLocationsCache( writable_locations, readable_locations) # If there are no readable_locations, we use the write endpoint as ReadEndpoint self.assertEqual(write_endpoint, Test_globaldb_tests.write_location_host) self.assertEqual(read_endpoint, Test_globaldb_tests.write_location_host) writable_locations = [] readable_locations = [{ 'name': Test_globaldb_tests.read_location, 'databaseAccountEndpoint': Test_globaldb_tests.read_location_host }] write_endpoint, read_endpoint = client._global_endpoint_manager.UpdateLocationsCache( writable_locations, readable_locations) # If there are no writable_locations, both Read and Write Endpoints point to endpoint passed while creating the client instance self.assertEqual(write_endpoint, Test_globaldb_tests.host) self.assertEqual(read_endpoint, Test_globaldb_tests.host) writable_locations = [{ 'name': Test_globaldb_tests.write_location, 'databaseAccountEndpoint': Test_globaldb_tests.write_location_host }] readable_locations = [{ 'name': Test_globaldb_tests.read_location, 'databaseAccountEndpoint': Test_globaldb_tests.read_location_host }, { 'name': Test_globaldb_tests.read_location2, 'databaseAccountEndpoint': Test_globaldb_tests.read_location2_host }] connection_policy = documents.ConnectionPolicy() connection_policy.PreferredLocations = [ Test_globaldb_tests.read_location2 ] client = cosmos_client_connection.CosmosClientConnection( Test_globaldb_tests.host, {'masterKey': Test_globaldb_tests.masterKey}, connection_policy) write_endpoint, read_endpoint = client._global_endpoint_manager.UpdateLocationsCache( writable_locations, readable_locations) # Test that the preferred location is set as ReadEndpoint instead of default write endpoint when no preference is set self.assertEqual(write_endpoint, Test_globaldb_tests.write_location_host) self.assertEqual(read_endpoint, Test_globaldb_tests.read_location2_host) writable_locations = [{ 'name': Test_globaldb_tests.write_location, 'databaseAccountEndpoint': Test_globaldb_tests.write_location_host }, { 'name': Test_globaldb_tests.read_location2, 'databaseAccountEndpoint': Test_globaldb_tests.read_location2_host }] readable_locations = [{ 'name': Test_globaldb_tests.read_location, 'databaseAccountEndpoint': Test_globaldb_tests.read_location_host }] connection_policy = documents.ConnectionPolicy() connection_policy.PreferredLocations = [ Test_globaldb_tests.read_location2 ] client = cosmos_client_connection.CosmosClientConnection( Test_globaldb_tests.host, {'masterKey': Test_globaldb_tests.masterKey}, connection_policy) write_endpoint, read_endpoint = client._global_endpoint_manager.UpdateLocationsCache( writable_locations, readable_locations) # Test that the preferred location is chosen from the WriteLocations if it's not present in the ReadLocations self.assertEqual(write_endpoint, Test_globaldb_tests.write_location_host) self.assertEqual(read_endpoint, Test_globaldb_tests.read_location2_host) writable_locations = [{ 'name': Test_globaldb_tests.write_location, 'databaseAccountEndpoint': Test_globaldb_tests.write_location_host }] readable_locations = [{ 'name': Test_globaldb_tests.read_location, 'databaseAccountEndpoint': Test_globaldb_tests.read_location_host }, { 'name': Test_globaldb_tests.read_location2, 'databaseAccountEndpoint': Test_globaldb_tests.read_location2_host }] connection_policy.EnableEndpointDiscovery = False client = cosmos_client_connection.CosmosClientConnection( Test_globaldb_tests.host, {'masterKey': Test_globaldb_tests.masterKey}, connection_policy) write_endpoint, read_endpoint = client._global_endpoint_manager.UpdateLocationsCache( writable_locations, readable_locations) # If EnableEndpointDiscovery is False, both Read and Write Endpoints point to endpoint passed while creating the client instance self.assertEqual(write_endpoint, Test_globaldb_tests.host) self.assertEqual(read_endpoint, Test_globaldb_tests.host)