def setUpClass(cls):
super(TestControllerServiceWithZoneTopology, cls).setUpClass()
cls.clusters = create_clusters(num_of_clusters=3,
num_of_client_per_cluster=3,
name_prefix='zone_')
for c in cls.clusters:
c.start()
for c in cls.clusters:
c.wait_until_is_alive()
cls.topology = get_config_topology_from_cluster_list(cls.clusters)
config = {
'MANAGEMENT_SERVERS': None,
'MANAGEMENT_PROTOCOL': None,
'MANAGEMENT_USERNAME': None,
'MANAGEMENT_PASSWORD': None,
'TOPOLOGY_TYPE': Consts.TopologyType.MULTIPLE_NVMESH_CLUSTERS,
'TOPOLOGY': cls.topology,
'LOG_LEVEL': 'DEBUG',
'SDK_LOG_LEVEL': 'DEBUG'
}
ConfigLoaderMock(config).load()
cls.driver_server = start_server(Consts.DriverType.Controller,
config=config)
cls.ctrl_client = ControllerClient()
def _start_csi_controller(self, config):
log.debug('Starting CSI Controller')
driver_server = start_server(Consts.DriverType.Controller,
config=config)
def stop_driver():
log.debug('stopping driver')
driver_server.stop()
self.addCleanup(stop_driver)
def test_retry_on_another_zone(self):
only_zone_c_active = {
"zones": {
"zone_1": {
"management": {
"servers": "some-unavailable-server-1:4000"
}
},
"zone_2": {
"management": {
"servers": "some-unavailable-server-2:4000"
}
},
"zone_3": {
"management": {
"servers": self.active_mgmt_server
}
},
"zone_4": {
"management": {
"servers": "some-unavailable-server-4:4000"
}
},
}
}
zones_1_to_4 = [
Topology(segments={Consts.TopologyKey.ZONE: 'zone_1'}),
Topology(segments={Consts.TopologyKey.ZONE: 'zone_2'}),
Topology(segments={Consts.TopologyKey.ZONE: 'zone_3'}),
Topology(segments={Consts.TopologyKey.ZONE: 'zone_4'})
]
volume_topology = TopologyRequirement(requisite=zones_1_to_4,
preferred=zones_1_to_4)
config = {
'TOPOLOGY_TYPE': Consts.TopologyType.MULTIPLE_NVMESH_CLUSTERS,
'TOPOLOGY': only_zone_c_active,
'LOG_LEVEL': 'INFO',
'SDK_LOG_LEVEL': 'INFO'
}
ConfigLoaderMock(config).load()
driver_server = start_server(Consts.DriverType.Controller,
config=config)
parameters = {'vpg': 'DEFAULT_CONCATENATED_VPG'}
ctrl_client = ControllerClient()
res = ctrl_client.CreateVolume(name=VOL_2_ID,
capacity_in_bytes=1 * GB,
parameters=parameters,
topology_requirements=volume_topology)
driver_server.stop()
self.assertTrue(res.volume.volume_id.startswith('zone_3:'))
def test_fail_available_zones_not_in_allowed_topology(self):
all_inactive = {
"zones": {
"A": {
"management": {
"servers": "some-unavailable-server-1:4000"
}
},
"B": {
"management": {
"servers": TestRetryOnAnotherZone.active_mgmt_server
}
},
"C": {
"management": {
"servers": "some-unavailable-server-3:4000"
}
}
}
}
requirement = TopologyRequirement(
requisite=[
Topology(segments={Consts.TopologyKey.ZONE: 'A'}),
Topology(segments={Consts.TopologyKey.ZONE: 'C'})
],
preferred=[
Topology(segments={Consts.TopologyKey.ZONE: 'A'}),
Topology(segments={Consts.TopologyKey.ZONE: 'C'})
])
config = {
'TOPOLOGY_TYPE': Consts.TopologyType.MULTIPLE_NVMESH_CLUSTERS,
'TOPOLOGY': all_inactive,
'LOG_LEVEL': 'DEBUG',
'SDK_LOG_LEVEL': 'INFO'
}
ConfigLoaderMock(config).load()
driver_server = start_server(Consts.DriverType.Controller,
config=config)
self.addCleanup(lambda: driver_server.stop())
with self.assertRaises(_Rendezvous):
parameters = {'vpg': 'DEFAULT_CONCATENATED_VPG'}
ctrl_client = ControllerClient()
res = ctrl_client.CreateVolume(name=VOL_2_ID,
capacity_in_bytes=1 * GB,
parameters=parameters,
topology_requirements=requirement)
def test_disable_zone_single_allowed_zone(self):
single_inactive = {
"zones": {
"A": {
"management": {
"servers": "some-unavailable-server-1:4000"
}
}
}
}
single_zone = [Topology(segments={Consts.TopologyKey.ZONE: 'A'})]
requirement = TopologyRequirement(requisite=single_zone,
preferred=single_zone)
config = {
'TOPOLOGY_TYPE': Consts.TopologyType.MULTIPLE_NVMESH_CLUSTERS,
'TOPOLOGY': single_inactive,
'LOG_LEVEL': 'DEBUG',
'SDK_LOG_LEVEL': 'INFO'
}
ConfigLoaderMock(config).load()
driver_server = start_server(Consts.DriverType.Controller,
config=config)
self.addCleanup(lambda: driver_server.stop())
parameters = {'vpg': 'DEFAULT_CONCATENATED_VPG'}
ctrl_client = ControllerClient()
def assert_fail_to_create_volume(volume_id):
try:
res = ctrl_client.CreateVolume(
name=volume_id,
capacity_in_bytes=1 * GB,
parameters=parameters,
topology_requirements=requirement)
self.addCleanup(
lambda: ctrl_client.DeleteVolume(volume_id=res.volume_id))
except _Rendezvous as ex:
self.assertEquals(ex._state.code,
StatusCode.RESOURCE_EXHAUSTED)
self.assertIn('Failed to create volume on all zones',
ex.debug_error_string())
assert_fail_to_create_volume(VOL_1_ID)
assert_fail_to_create_volume(VOL_2_ID)
def test_simple_termination(self):
config = {
'MANAGEMENT_SERVERS': 'localhost:4000',
'MANAGEMENT_PROTOCOL': 'https',
'MANAGEMENT_USERNAME': '******',
'MANAGEMENT_PASSWORD': '******',
'TOPOLOGY_TYPE': Consts.TopologyType.SINGLE_ZONE_CLUSTER,
'TOPOLOGY': None
}
driver_server = start_server(Consts.DriverType.Controller,
config=config,
wait_for_grpc=False)
time.sleep(3)
driver_server.stop()
def test_abort_during_request(self):
cluster = NVMeshCluster('cluster1',
options={'volumeCreationDelayMS': 5000})
cluster.start()
self.addCleanup(lambda: cluster.stop())
config = {
'MANAGEMENT_SERVERS': 'localhost:4000',
'MANAGEMENT_PROTOCOL': 'https',
'MANAGEMENT_USERNAME': '******',
'MANAGEMENT_PASSWORD': '******',
'TOPOLOGY_TYPE': Consts.TopologyType.SINGLE_ZONE_CLUSTER,
'TOPOLOGY': None
}
ConfigLoaderMock(config).load()
driver_server = start_server(Consts.DriverType.Controller,
config=config)
response_bucket = {}
response_bucket['volume_id'] = None
def create_volume(response_bucket):
ctrl_client = ControllerClient()
parameters = {'vpg': 'DEFAULT_CONCATENATED_VPG'}
res = ctrl_client.CreateVolume(
name='pvc-test-graceful-shutdown',
capacity_in_bytes=5 * GB,
parameters=parameters,
topology_requirements=TOPO_REQ_MULTIPLE_TOPOLOGIES)
log.debug('create_volume returned %s' % res)
response_bucket['volume_id'] = res.volume.volume_id
t = threading.Thread(target=create_volume, args=(response_bucket, ))
t.start()
time.sleep(2)
log.debug('shutting the server down')
driver_server.stop()
# if volume_id is None that means the thread pre-maturely terminated
self.assertTrue(response_bucket['volume_id'])
def setUpClass(cls):
super(TestControllerServiceWithoutTopology, cls).setUpClass()
cls.cluster1 = NVMeshCluster('cluster_' + cls.__name__)
cls.cluster1.start()
config = {
'MANAGEMENT_SERVERS': cls.cluster1.get_mgmt_server_string(),
'MANAGEMENT_PROTOCOL': 'https',
'MANAGEMENT_USERNAME': '******',
'MANAGEMENT_PASSWORD': '******',
'TOPOLOGY_TYPE': None,
'TOPOLOGY': None,
'SDK_LOG_LEVEL': 'DEBUG'
}
ConfigLoaderMock(config).load()
cls.driver_server = start_server(Consts.DriverType.Controller,
config=config)
cls.ctrl_client = ControllerClient()
def test_fail_to_create_all_mgmts_not_available(self):
all_inactive = {
"zones": {
"zone_1": {
"management": {
"servers": "some-unavailable-server-1:4000"
}
},
"zone_2": {
"management": {
"servers": "some-unavailable-server-2:4000"
}
},
"zone_3": {
"management": {
"servers": "some-unavailable-server-3:4000"
}
}
}
}
config = {
'TOPOLOGY_TYPE': Consts.TopologyType.MULTIPLE_NVMESH_CLUSTERS,
'TOPOLOGY': all_inactive,
'LOG_LEVEL': 'DEBUG',
'SDK_LOG_LEVEL': 'INFO'
}
ConfigLoaderMock(config).load()
driver_server = start_server(Consts.DriverType.Controller,
config=config)
self.addCleanup(lambda: driver_server.stop())
with self.assertRaises(_Rendezvous):
parameters = {'vpg': 'DEFAULT_CONCATENATED_VPG'}
ctrl_client = ControllerClient()
res = ctrl_client.CreateVolume(
name=VOL_2_ID,
capacity_in_bytes=1 * GB,
parameters=parameters,
topology_requirements=TOPO_REQ_MULTIPLE_TOPOLOGIES)