def test_should_not_raise_any_exception_when_switching_to_another_cluster_version(self):
        senza_mock = SenzaWrapper(CONFIG)
        senza_switch_mock = senza_mock.switch_traffic = MagicMock()
        senza_passive_versions_mock = senza_mock.get_passive_stack_version = MagicMock(return_value='test-version')

        controller = ClusterDeploymentController(base_url=BASE_URL, stack_name=STACK_NAME,
                                                 image_version=IMAGE_VERSION, oauth_token=OAUTH_TOKEN,
                                                 senza_wrapper=senza_mock)
        controller.switch_traffic()

        senza_passive_versions_mock.assert_called_once_with(STACK_NAME)
        senza_switch_mock.assert_called_once_with(STACK_NAME, 'test-version', 100)
    def test_should_return_green_when_only_blue_stack_is_running_and_passive_stack_version_is_requested(self):

        active_version = 'blue'
        senza_mock = SenzaWrapper(CONFIG)
        senza_passive_versions_mock = senza_mock.get_passive_stack_version = MagicMock(return_value=None)
        senza_active_versions_mock = senza_mock.get_active_stack_version = MagicMock(return_value=active_version)

        controller = ClusterDeploymentController(base_url=BASE_URL, stack_name=STACK_NAME,
                                                 image_version=IMAGE_VERSION, oauth_token=OAUTH_TOKEN,
                                                 senza_wrapper=senza_mock)

        self.assertEquals('green', controller.get_passive_stack_version(), 'Wrong stack version')

        senza_passive_versions_mock.assert_called_once_with(STACK_NAME)
        senza_active_versions_mock.assert_called_once_with(STACK_NAME)
    def test_should_raise_exception_on_timeout_when_creating_a_new_cluster(self):
        instances = [
            NEW_NODES
        ]
        senza_mock = SenzaWrapper(CONFIG)
        senza_create_mock = senza_mock.create_stack = MagicMock()
        senza_passive_versions_mock = senza_mock.get_passive_stack_version = MagicMock(return_value='test-version')
        senza_instances_mock = senza_mock.get_stack_instances = MagicMock(side_effect=instances)

        http_calls = [
            self.__side_effect_return_cluster_state_old_nodes(None),
            self.__side_effect_return_cluster_state_old_nodes(None),
        ]

        urlopen_mock = MagicMock(side_effect=http_calls)
        urllib.request.urlopen = urlopen_mock

        controller = ClusterDeploymentController(base_url=BASE_URL, stack_name=STACK_NAME,
                                                 image_version=IMAGE_VERSION, oauth_token=OAUTH_TOKEN,
                                                 senza_wrapper=senza_mock)
        controller.set_create_cluster_retry_wait(0)
        controller.set_create_cluster_timeout(0)
        with self.assertRaisesRegex(Exception, 'Timeout while creating new cluster, not all new nodes have been '
                                               'registered in time'):
            controller.create_cluster()

        senza_passive_versions_mock.assert_called_with(STACK_NAME)
        self.assertEquals(2, len(senza_passive_versions_mock.call_args_list))
        senza_create_mock.assert_called_once_with(STACK_NAME, 'test-version', IMAGE_VERSION)
        senza_instances_mock.assert_called_once_with(STACK_NAME, 'test-version')
 def setUp(self):
     urllib.request.urlopen = MagicMock(side_effect=self.__side_effect_return_cluster_state_old_nodes)
     senza_wrapper = SenzaWrapper(CONFIG)
     self.__controller = ClusterDeploymentController(base_url=BASE_URL, stack_name=STACK_NAME,
                                                     image_version=IMAGE_VERSION, oauth_token=OAUTH_TOKEN,
                                                     senza_wrapper=senza_wrapper)
     self.__controller.set_leader_check_retry_count(1)
     self.__controller.set_leader_check_retry_wait(0)
     self.__controller.set_add_node_retry_count(1)
     self.__controller.set_add_node_retry_wait(0)
     self.__controller.set_add_node_timeout(1)
     self.__controller.set_create_cluster_retry_wait(0)
     self.__controller.set_create_cluster_timeout(1)
    def test_should_not_raise_any_exception_when_creating_a_new_cluster(self):
        instances = [
            NEW_NODES
        ]
        senza_mock = SenzaWrapper(CONFIG)
        senza_create_mock = senza_mock.create_stack = MagicMock()
        senza_passive_versions_mock = senza_mock.get_passive_stack_version = MagicMock(return_value='test-version')
        senza_instances_mock = senza_mock.get_stack_instances = MagicMock(side_effect=instances)

        http_calls = self.__side_effect_return_cluster_state_all_registered_nodes

        urlopen_mock = MagicMock(side_effect=http_calls)
        urllib.request.urlopen = urlopen_mock

        controller = ClusterDeploymentController(base_url=BASE_URL, stack_name=STACK_NAME,
                                                 image_version=IMAGE_VERSION, oauth_token=OAUTH_TOKEN,
                                                 senza_wrapper=senza_mock)
        controller.create_cluster()

        senza_passive_versions_mock.assert_called_with(STACK_NAME)
        self.assertEquals(2, len(senza_passive_versions_mock.call_args_list))
        senza_create_mock.assert_called_once_with(STACK_NAME, 'test-version', IMAGE_VERSION)
        senza_instances_mock.assert_called_once_with(STACK_NAME, 'test-version')
    def test_should_not_raise_exceptions_when_executing_all_deployment_steps(self):
        test_version = 'test-version'
        instances = [
            OLD_NODES,
            NEW_NODES,
            OLD_NODES
        ]
        senza_mock = SenzaWrapper(CONFIG)
        senza_create_mock = senza_mock.create_stack = MagicMock()
        senza_passive_versions_mock = senza_mock.get_passive_stack_version = MagicMock(return_value=test_version)
        senza_switch_mock = senza_mock.switch_traffic = MagicMock(return_value=True)
        senza_delete_mock = senza_mock.delete_stack_version = MagicMock()
        senza_instances_mock = senza_mock.get_stack_instances = MagicMock(side_effect=instances)

        controller = ClusterDeploymentController(base_url=BASE_URL, stack_name=STACK_NAME,
                                                 image_version=IMAGE_VERSION, oauth_token=OAUTH_TOKEN,
                                                 senza_wrapper=senza_mock)

        controller.set_leader_check_retry_count(1)
        controller.set_leader_check_retry_wait(0)
        controller.set_add_node_retry_count(1)
        controller.set_add_node_retry_wait(0)
        controller.set_add_node_timeout(1)

        http_calls = [
            self.__side_effect_return_cluster_state_all_registered_nodes(None),  # create_cluster
            self.__side_effect_return_cluster_state_all_registered_nodes(None),  # add_new_nodes_to_cluster
            self.__side_effect_all_ok(''),  # add_new_nodes_to_cluster
            self.__side_effect_all_ok(''),  # add_new_nodes_to_cluster
            self.__side_effect_all_ok(''),  # add_new_nodes_to_cluster
            self.__side_effect_return_cluster_state_all_nodes(None),  # add_new_nodes_to_cluster
            self.__side_effect_return_cluster_state_all_nodes(None),  # delete_old_nodes_from_cluster
            self.__side_effect_return_cluster_state_all_nodes(None),  # delete_old_nodes_from_cluster
            self.__side_effect_all_ok(''),  # delete_old_nodes_from_cluster
            self.__side_effect_return_cluster_state_all_nodes(None),  # delete_old_nodes_from_cluster
            self.__side_effect_all_ok(''),  # delete_old_nodes_from_cluster
            self.__side_effect_return_cluster_state_all_nodes(None),  # delete_old_nodes_from_cluster
            self.__side_effect_all_ok(''),  # delete_old_nodes_from_cluster
            self.__side_effect_return_cluster_state_all_nodes(None),  # delete_old_nodes_from_cluster
        ]

        urlopen_mock = MagicMock(side_effect=http_calls)
        urllib.request.urlopen = urlopen_mock

        controller.deploy_new_version()

        senza_passive_versions_mock.assert_called_with(STACK_NAME)
        senza_create_mock.assert_called_once_with(STACK_NAME, test_version, IMAGE_VERSION)
        senza_switch_mock.assert_called_once_with(STACK_NAME, test_version, 100)
        senza_delete_mock.assert_called_once_with(STACK_NAME, test_version)
        senza_instances_mock.assert_called_with(STACK_NAME, test_version)
 def test_should_return_zero_if_shard_has_no_replicas(self):
     collection = list(CLUSTER_INACTIVE_SHARD_NO_REPLICAS['cluster']['collections'].keys())[0]
     shard = list(CLUSTER_INACTIVE_SHARD_NO_REPLICAS['cluster']['collections'][collection]['shards'].keys())[0]
     self.assertEquals(0, ClusterDeploymentController.get_number_of_active_nodes(CLUSTER_INACTIVE_SHARD_NO_REPLICAS,
                                                                                 collection, shard),
                       'Shard should have no replicas')
 def test_should_return_failure_because_of_inactive_shard_in_cluster(self):
     collection = list(CLUSTER_INACTIVE_SHARD_NO_REPLICAS['cluster']['collections'].keys())[0]
     shard = list(CLUSTER_INACTIVE_SHARD_NO_REPLICAS['cluster']['collections'][collection]['shards'].keys())[0]
     self.assertFalse(ClusterDeploymentController.has_active_leader(CLUSTER_INACTIVE_SHARD_NO_REPLICAS, collection,
                                                                    shard),
                      'Cluster should have no active shards')
class TestClusterDeploymentController(TestCase):

    __controller = None
    __urlopen_mock = None

    def setUp(self):
        urllib.request.urlopen = MagicMock(side_effect=self.__side_effect_return_cluster_state_old_nodes)
        senza_wrapper = SenzaWrapper(CONFIG)
        self.__controller = ClusterDeploymentController(base_url=BASE_URL, stack_name=STACK_NAME,
                                                        image_version=IMAGE_VERSION, oauth_token=OAUTH_TOKEN,
                                                        senza_wrapper=senza_wrapper)
        self.__controller.set_leader_check_retry_count(1)
        self.__controller.set_leader_check_retry_wait(0)
        self.__controller.set_add_node_retry_count(1)
        self.__controller.set_add_node_retry_wait(0)
        self.__controller.set_add_node_timeout(1)
        self.__controller.set_create_cluster_retry_wait(0)
        self.__controller.set_create_cluster_timeout(1)

    def test_should_return_success_after_adding_new_replica_to_cluster(self):
        urllib.request.urlopen = MagicMock(side_effect=self.__side_effect_all_ok)
        self.assertEqual(0, self.__controller.add_replica_to_cluster('test', 'test', 'test'))

    def test_should_return_failure_because_of_unknown_status_code_from_solr_when_adding_new_replica_to_cluster(self):
        urllib.request.urlopen = MagicMock(side_effect=self.__side_effect_unknown_http_code)
        with self.assertRaisesRegex(Exception, 'Received unexpected status code from Solr: \[{}\]'
                                               .format(str(HTTP_CODE_UNKNOWN))):
            self.__controller.add_replica_to_cluster('test', 'test', 'test')

    def test_should_return_success_after_timeout_when_adding_new_replica_to_cluster(self):
        urllib.request.urlopen = MagicMock(side_effect=self.__side_effect_timeout)
        self.assertEqual(0, self.__controller.add_replica_to_cluster('test', 'test', 'test'))

    def test_should_return_success_after_a_bad_request_error_followed_by_a_ok_on_the_retry_when_adding_replica(self):
        response_mock = MagicMock()
        response_mock.getcode.return_value = HTTP_CODE_OK
        side_effects = [
            urllib.error.HTTPError(url=None, code=HTTP_CODE_BAD_REQUEST, msg=None, hdrs=None, fp=None),
            response_mock
        ]
        urllib.request.urlopen = MagicMock(side_effect=side_effects)
        self.assertEqual(0, self.__controller.add_replica_to_cluster('test', 'test', 'test'))

    def test_should_return_failure_because_of_unknown_http_error_when_adding_new_replica_to_cluster(self):
        urllib.request.urlopen = MagicMock(side_effect=self.__side_effect_unknown_http_error)
        with self.assertRaisesRegex(Exception, 'Failed sending request to Solr \[{}\]: HTTP Error {}: .*'
                                               .format(API_URL + '[^\]]+', str(HTTP_CODE_UNKNOWN_ERROR))):
            self.__controller.add_replica_to_cluster('test', 'test', 'test')

    def test_should_return_success_after_adding_multiple_nodes_to_cluster(self):
        cluster_states = [
            self.__side_effect_return_cluster_state_all_registered_nodes(None),
            self.__side_effect_all_ok(''),
            self.__side_effect_all_ok(''),
            self.__side_effect_all_ok(''),
            self.__side_effect_return_cluster_state_all_nodes(None)
        ]
        urlopen_mock = MagicMock(side_effect=cluster_states)
        urllib.request.urlopen = urlopen_mock
        collection = list(CLUSTER_OLD_NODES['cluster']['collections'].keys())[0]
        shard = list(CLUSTER_OLD_NODES['cluster']['collections'][collection]['shards'].keys())[0]
        urls = list()
        for node_ip in NEW_NODES:
            node = node_ip + ':8983_solr'
            url = API_URL + '?action=ADDREPLICA'
            url += '&collection=' + collection
            url += '&shard=' + shard
            url += '&node=' + node
            urls.append(url)

        senza_mock = MagicMock()
        senza_mock.get_stack_instances.return_value = NEW_NODES
        self.__controller.set_senza_wrapper(senza_mock)

        self.__controller.add_new_nodes_to_cluster()

        called_urls = list(map(lambda x: x[0][0].get_full_url(), urlopen_mock.call_args_list))
        for url in urls:
            self.assertIn(url, called_urls, 'URL was not called')

    def test_should_return_error_because_of_not_enough_nodes_for_cluster_layout(self):
        urllib.request.urlopen = MagicMock(side_effect=self.__side_effect_return_cluster_state_old_nodes)
        senza_mock = MagicMock()
        senza_mock.get_stack_instances.return_value = NOT_ENOUGH_NODES
        self.__controller.set_senza_wrapper(senza_mock)
        with self.assertRaises(Exception, msg='Not enough instances for current cluster layout: [2]<[3]'):
            self.__controller.add_new_nodes_to_cluster()

    def test_should_wait_until_all_nodes_are_active(self):
        http_calls = [
            self.__side_effect_return_cluster_state_all_registered_nodes(None),
            self.__side_effect_all_ok(''),
            self.__side_effect_all_ok(''),
            self.__side_effect_all_ok(''),
            self.__side_effect_return_cluster_state_all_nodes_one_not_active(None),
            self.__side_effect_return_cluster_state_all_nodes(None)
        ]
        urlopen_mock = MagicMock(side_effect=http_calls)
        urllib.request.urlopen = urlopen_mock
        collection = list(CLUSTER_OLD_NODES['cluster']['collections'].keys())[0]
        shard = list(CLUSTER_OLD_NODES['cluster']['collections'][collection]['shards'].keys())[0]
        urls = list()
        for node_ip in NEW_NODES:
            node = node_ip + ':8983_solr'
            url = API_URL + '?action=ADDREPLICA'
            url += '&collection=' + collection
            url += '&shard=' + shard
            url += '&node=' + node
            urls.append(url)

        senza_mock = MagicMock()
        senza_mock.get_stack_instances.return_value = NEW_NODES
        self.__controller.set_senza_wrapper(senza_mock)

        self.__controller.add_new_nodes_to_cluster()

        called_urls = list(map(lambda x: x[0][0].get_full_url(), urlopen_mock.call_args_list))
        for url in urls:
            self.assertIn(url, called_urls, 'URL was not called')

    def test_should_raise_exception_on_timeout_when_adding_nodes(self):
        http_calls = [
            self.__side_effect_return_cluster_state_all_registered_nodes(None),
            self.__side_effect_all_ok(''),
            self.__side_effect_all_ok(''),
            self.__side_effect_all_ok(''),
        ]
        urlopen_mock = MagicMock(side_effect=http_calls)
        urllib.request.urlopen = urlopen_mock
        collection = list(CLUSTER_OLD_NODES['cluster']['collections'].keys())[0]
        shard = list(CLUSTER_OLD_NODES['cluster']['collections'][collection]['shards'].keys())[0]
        urls = list()
        for node_ip in NEW_NODES:
            node = node_ip + ':8983_solr'
            url = API_URL + '?action=ADDREPLICA'
            url += '&collection=' + collection
            url += '&shard=' + shard
            url += '&node=' + node
            urls.append(url)

        senza_mock = MagicMock()
        senza_mock.get_stack_instances.return_value = NEW_NODES
        self.__controller.set_senza_wrapper(senza_mock)
        self.__controller.set_add_node_timeout(0)

        with self.assertRaisesRegex(Exception, 'Timeout while adding new nodes to cluster'):
            self.__controller.add_new_nodes_to_cluster()

        called_urls = list(map(lambda x: x[0][0].get_full_url(), urlopen_mock.call_args_list))
        for url in urls:
            self.assertIn(url, called_urls, 'URL was not called')

    def test_should_not_raise_exception_when_deleting_replica_from_cluster(self):
        urllib.request.urlopen = MagicMock(side_effect=self.__side_effect_all_ok)
        self.__controller.delete_replica_from_cluster('test', 'test', 'test')

    def test_should_return_failure_because_of_unknown_status_code_from_solr_when_deleting_replica_from_cluster(self):
        urllib.request.urlopen = MagicMock(side_effect=self.__side_effect_unknown_http_code)
        with self.assertRaisesRegex(Exception, 'Received unexpected status code from Solr: \[{}\]'
                                               .format(str(HTTP_CODE_UNKNOWN))):
            self.__controller.delete_replica_from_cluster('test', 'test', 'test')

    def test_should_not_raise_exception_after_timeout_when_deleting_replica_from_cluster(self):
        urllib.request.urlopen = MagicMock(side_effect=self.__side_effect_timeout)
        self.__controller.delete_replica_from_cluster('test', 'test', 'test')

    def test_should_not_raise_exception_after_error_when_deleting_replica_from_cluster(self):
        urllib.request.urlopen = MagicMock(side_effect=self.__side_effect_error)
        self.__controller.delete_replica_from_cluster('test', 'test', 'test')

    def test_should_return_failure_because_of_unknown_http_error_when_deleting_replica_from_cluster(self):
        urllib.request.urlopen = MagicMock(side_effect=self.__side_effect_unknown_http_error)
        with self.assertRaisesRegex(Exception, 'Failed sending request to Solr \[{}\]: HTTP Error {}: .*'
                                               .format(API_URL + '[^\]]+', str(HTTP_CODE_UNKNOWN_ERROR))):
            self.__controller.delete_replica_from_cluster('test', 'test', 'test')

    def test_should_return_success_after_deleting_multiple_nodes_from_cluster(self):
        urlopen_mock = MagicMock(side_effect=self.__side_effect_return_cluster_state_old_nodes)
        urllib.request.urlopen = urlopen_mock
        collection = list(CLUSTER_OLD_NODES['cluster']['collections'].keys())[0]
        shard = list(CLUSTER_OLD_NODES['cluster']['collections'][collection]['shards'].keys())[0]
        replicas = list(CLUSTER_OLD_NODES['cluster']['collections'][collection]['shards'][shard]['replicas'].keys())
        urls = list()
        for replica in replicas:
            url = API_URL + '?action=DELETEREPLICA'
            url += '&collection=' + collection
            url += '&shard=' + shard
            url += '&replica=' + replica
            urls.append(url)
        senza_mock = MagicMock()
        senza_mock.get_stack_instances.return_value = OLD_NODES
        self.__controller.set_senza_wrapper(senza_mock)

        self.__controller.delete_old_nodes_from_cluster()
        called_urls = list(map(lambda x: x[0][0].get_full_url(), urlopen_mock.call_args_list))
        for url in urls:
            self.assertIn(url, called_urls, 'URL was not called')

    def test_should_return_failure_after_deleting_multiple_nodes_from_shard_without_leader(self):
        urlopen_mock = MagicMock(side_effect=self.__side_effect_return_cluster_state_no_leader)
        urllib.request.urlopen = urlopen_mock
        senza_mock = MagicMock()
        senza_mock.get_stack_instances.return_value = OLD_NODES
        self.__controller.set_senza_wrapper(senza_mock)
        with self.assertRaisesRegex(Exception, 'Shard \[shard[0-9]\] of collection \[[a-z0-9\-]+\] has no active '
                                               'leader'):
            self.__controller.delete_old_nodes_from_cluster()

    def test_should_return_failure_after_deleting_multiple_nodes_with_only_one_active_replica(self):
        urlopen_mock = MagicMock(side_effect=self.__side_effect_return_cluster_state_only_one_active_replica)
        urllib.request.urlopen = urlopen_mock
        senza_mock = MagicMock()
        senza_mock.get_stack_instances.return_value = OLD_NODES
        self.__controller.set_senza_wrapper(senza_mock)
        with self.assertRaisesRegex(Exception, 'Shard \[shard[0-9]\] of collection \[[a-z0-9\-]+\] has not enough '
                                               'active nodes: \[[1]\]'):
            self.__controller.delete_old_nodes_from_cluster()

    def test_should_return_current_cluster_state(self):
        urllib.request.urlopen = MagicMock(side_effect=self.__side_effect_return_cluster_state_old_nodes)
        cluster_state = self.__controller.get_cluster_state()
        self.assertEqual(CLUSTER_OLD_NODES, cluster_state, 'Cluster state does not match test cluster')

    def test_should_return_failure_because_of_unknown_status_code_from_solr_when_requesting_cluster_state(self):
        urllib.request.urlopen = MagicMock(side_effect=self.__side_effect_unknown_http_code)
        with self.assertRaisesRegex(Exception, 'Received unexpected status code from Solr: \[{}\]'
                                               .format(str(HTTP_CODE_UNKNOWN))):
            self.__controller.get_cluster_state()

    def test_should_return_failure_because_of_unknown_http_error_when_requesting_cluster_state(self):
        urllib.request.urlopen = MagicMock(side_effect=self.__side_effect_unknown_http_error)
        with self.assertRaisesRegex(Exception, 'Failed sending request to Solr \[{}\]: HTTP Error {}: .*'
                                               .format(API_URL + '[^\]]+', str(HTTP_CODE_UNKNOWN_ERROR))):
            self.__controller.get_cluster_state()

    def test_should_return_failure_because_of_inactive_shard_in_cluster(self):
        collection = list(CLUSTER_INACTIVE_SHARD_NO_REPLICAS['cluster']['collections'].keys())[0]
        shard = list(CLUSTER_INACTIVE_SHARD_NO_REPLICAS['cluster']['collections'][collection]['shards'].keys())[0]
        self.assertFalse(ClusterDeploymentController.has_active_leader(CLUSTER_INACTIVE_SHARD_NO_REPLICAS, collection,
                                                                       shard),
                         'Cluster should have no active shards')

    def test_should_return_zero_if_shard_has_no_replicas(self):
        collection = list(CLUSTER_INACTIVE_SHARD_NO_REPLICAS['cluster']['collections'].keys())[0]
        shard = list(CLUSTER_INACTIVE_SHARD_NO_REPLICAS['cluster']['collections'][collection]['shards'].keys())[0]
        self.assertEquals(0, ClusterDeploymentController.get_number_of_active_nodes(CLUSTER_INACTIVE_SHARD_NO_REPLICAS,
                                                                                    collection, shard),
                          'Shard should have no replicas')

    def test_should_return_green_when_only_blue_stack_is_running_and_passive_stack_version_is_requested(self):

        active_version = 'blue'
        senza_mock = SenzaWrapper(CONFIG)
        senza_passive_versions_mock = senza_mock.get_passive_stack_version = MagicMock(return_value=None)
        senza_active_versions_mock = senza_mock.get_active_stack_version = MagicMock(return_value=active_version)

        controller = ClusterDeploymentController(base_url=BASE_URL, stack_name=STACK_NAME,
                                                 image_version=IMAGE_VERSION, oauth_token=OAUTH_TOKEN,
                                                 senza_wrapper=senza_mock)

        self.assertEquals('green', controller.get_passive_stack_version(), 'Wrong stack version')

        senza_passive_versions_mock.assert_called_once_with(STACK_NAME)
        senza_active_versions_mock.assert_called_once_with(STACK_NAME)

    def test_should_not_raise_any_exception_when_creating_a_new_cluster(self):
        instances = [
            NEW_NODES
        ]
        senza_mock = SenzaWrapper(CONFIG)
        senza_create_mock = senza_mock.create_stack = MagicMock()
        senza_passive_versions_mock = senza_mock.get_passive_stack_version = MagicMock(return_value='test-version')
        senza_instances_mock = senza_mock.get_stack_instances = MagicMock(side_effect=instances)

        http_calls = self.__side_effect_return_cluster_state_all_registered_nodes

        urlopen_mock = MagicMock(side_effect=http_calls)
        urllib.request.urlopen = urlopen_mock

        controller = ClusterDeploymentController(base_url=BASE_URL, stack_name=STACK_NAME,
                                                 image_version=IMAGE_VERSION, oauth_token=OAUTH_TOKEN,
                                                 senza_wrapper=senza_mock)
        controller.create_cluster()

        senza_passive_versions_mock.assert_called_with(STACK_NAME)
        self.assertEquals(2, len(senza_passive_versions_mock.call_args_list))
        senza_create_mock.assert_called_once_with(STACK_NAME, 'test-version', IMAGE_VERSION)
        senza_instances_mock.assert_called_once_with(STACK_NAME, 'test-version')

    def test_should_wait_until_all_nodes_are_registered_when_creating_a_new_cluster(self):
        instances = [
            NEW_NODES
        ]
        senza_mock = SenzaWrapper(CONFIG)
        senza_create_mock = senza_mock.create_stack = MagicMock()
        senza_passive_versions_mock = senza_mock.get_passive_stack_version = MagicMock(return_value='test-version')
        senza_instances_mock = senza_mock.get_stack_instances = MagicMock(side_effect=instances)

        http_calls = [
            self.__side_effect_return_cluster_state_old_nodes(None),
            self.__side_effect_return_cluster_state_old_nodes(None),
            self.__side_effect_return_cluster_state_all_registered_nodes(None)
        ]

        urlopen_mock = MagicMock(side_effect=http_calls)
        urllib.request.urlopen = urlopen_mock

        controller = ClusterDeploymentController(base_url=BASE_URL, stack_name=STACK_NAME,
                                                 image_version=IMAGE_VERSION, oauth_token=OAUTH_TOKEN,
                                                 senza_wrapper=senza_mock)
        controller.set_create_cluster_retry_wait(0)
        controller.set_create_cluster_timeout(1)
        controller.create_cluster()

        senza_passive_versions_mock.assert_called_with(STACK_NAME)
        self.assertEquals(2, len(senza_passive_versions_mock.call_args_list))
        senza_create_mock.assert_called_once_with(STACK_NAME, 'test-version', IMAGE_VERSION)
        senza_instances_mock.assert_called_once_with(STACK_NAME, 'test-version')

    def test_should_raise_exception_on_timeout_when_creating_a_new_cluster(self):
        instances = [
            NEW_NODES
        ]
        senza_mock = SenzaWrapper(CONFIG)
        senza_create_mock = senza_mock.create_stack = MagicMock()
        senza_passive_versions_mock = senza_mock.get_passive_stack_version = MagicMock(return_value='test-version')
        senza_instances_mock = senza_mock.get_stack_instances = MagicMock(side_effect=instances)

        http_calls = [
            self.__side_effect_return_cluster_state_old_nodes(None),
            self.__side_effect_return_cluster_state_old_nodes(None),
        ]

        urlopen_mock = MagicMock(side_effect=http_calls)
        urllib.request.urlopen = urlopen_mock

        controller = ClusterDeploymentController(base_url=BASE_URL, stack_name=STACK_NAME,
                                                 image_version=IMAGE_VERSION, oauth_token=OAUTH_TOKEN,
                                                 senza_wrapper=senza_mock)
        controller.set_create_cluster_retry_wait(0)
        controller.set_create_cluster_timeout(0)
        with self.assertRaisesRegex(Exception, 'Timeout while creating new cluster, not all new nodes have been '
                                               'registered in time'):
            controller.create_cluster()

        senza_passive_versions_mock.assert_called_with(STACK_NAME)
        self.assertEquals(2, len(senza_passive_versions_mock.call_args_list))
        senza_create_mock.assert_called_once_with(STACK_NAME, 'test-version', IMAGE_VERSION)
        senza_instances_mock.assert_called_once_with(STACK_NAME, 'test-version')

    def test_should_not_raise_any_exception_when_deleting_a_cluster(self):
        senza_mock = SenzaWrapper(CONFIG)
        senza_delete_mock = senza_mock.delete_stack_version = MagicMock()
        senza_passive_versions_mock = senza_mock.get_passive_stack_version = MagicMock(return_value='test-version')

        controller = ClusterDeploymentController(base_url=BASE_URL, stack_name=STACK_NAME,
                                                 image_version=IMAGE_VERSION, oauth_token=OAUTH_TOKEN,
                                                 senza_wrapper=senza_mock)
        controller.delete_cluster()

        senza_passive_versions_mock.assert_called_once_with(STACK_NAME)
        senza_delete_mock.assert_called_once_with(STACK_NAME, 'test-version')

    def test_should_not_raise_any_exception_when_switching_to_another_cluster_version(self):
        senza_mock = SenzaWrapper(CONFIG)
        senza_switch_mock = senza_mock.switch_traffic = MagicMock()
        senza_passive_versions_mock = senza_mock.get_passive_stack_version = MagicMock(return_value='test-version')

        controller = ClusterDeploymentController(base_url=BASE_URL, stack_name=STACK_NAME,
                                                 image_version=IMAGE_VERSION, oauth_token=OAUTH_TOKEN,
                                                 senza_wrapper=senza_mock)
        controller.switch_traffic()

        senza_passive_versions_mock.assert_called_once_with(STACK_NAME)
        senza_switch_mock.assert_called_once_with(STACK_NAME, 'test-version', 100)

    def test_should_not_raise_exceptions_when_executing_all_deployment_steps(self):
        test_version = 'test-version'
        instances = [
            OLD_NODES,
            NEW_NODES,
            OLD_NODES
        ]
        senza_mock = SenzaWrapper(CONFIG)
        senza_create_mock = senza_mock.create_stack = MagicMock()
        senza_passive_versions_mock = senza_mock.get_passive_stack_version = MagicMock(return_value=test_version)
        senza_switch_mock = senza_mock.switch_traffic = MagicMock(return_value=True)
        senza_delete_mock = senza_mock.delete_stack_version = MagicMock()
        senza_instances_mock = senza_mock.get_stack_instances = MagicMock(side_effect=instances)

        controller = ClusterDeploymentController(base_url=BASE_URL, stack_name=STACK_NAME,
                                                 image_version=IMAGE_VERSION, oauth_token=OAUTH_TOKEN,
                                                 senza_wrapper=senza_mock)

        controller.set_leader_check_retry_count(1)
        controller.set_leader_check_retry_wait(0)
        controller.set_add_node_retry_count(1)
        controller.set_add_node_retry_wait(0)
        controller.set_add_node_timeout(1)

        http_calls = [
            self.__side_effect_return_cluster_state_all_registered_nodes(None),  # create_cluster
            self.__side_effect_return_cluster_state_all_registered_nodes(None),  # add_new_nodes_to_cluster
            self.__side_effect_all_ok(''),  # add_new_nodes_to_cluster
            self.__side_effect_all_ok(''),  # add_new_nodes_to_cluster
            self.__side_effect_all_ok(''),  # add_new_nodes_to_cluster
            self.__side_effect_return_cluster_state_all_nodes(None),  # add_new_nodes_to_cluster
            self.__side_effect_return_cluster_state_all_nodes(None),  # delete_old_nodes_from_cluster
            self.__side_effect_return_cluster_state_all_nodes(None),  # delete_old_nodes_from_cluster
            self.__side_effect_all_ok(''),  # delete_old_nodes_from_cluster
            self.__side_effect_return_cluster_state_all_nodes(None),  # delete_old_nodes_from_cluster
            self.__side_effect_all_ok(''),  # delete_old_nodes_from_cluster
            self.__side_effect_return_cluster_state_all_nodes(None),  # delete_old_nodes_from_cluster
            self.__side_effect_all_ok(''),  # delete_old_nodes_from_cluster
            self.__side_effect_return_cluster_state_all_nodes(None),  # delete_old_nodes_from_cluster
        ]

        urlopen_mock = MagicMock(side_effect=http_calls)
        urllib.request.urlopen = urlopen_mock

        controller.deploy_new_version()

        senza_passive_versions_mock.assert_called_with(STACK_NAME)
        senza_create_mock.assert_called_once_with(STACK_NAME, test_version, IMAGE_VERSION)
        senza_switch_mock.assert_called_once_with(STACK_NAME, test_version, 100)
        senza_delete_mock.assert_called_once_with(STACK_NAME, test_version)
        senza_instances_mock.assert_called_with(STACK_NAME, test_version)

    def test_should_not_raise_exception_if_shard_is_healthy(self):
        self.__controller.verify_shard_health(COLLECTION, SHARD)

    def test_should_raise_exception_if_shard_has_no_active_leader(self):
        urlopen_mock = MagicMock(side_effect=self.__side_effect_return_cluster_state_no_leader)
        urllib.request.urlopen = urlopen_mock
        with self.assertRaisesRegex(Exception, 'Shard \[{}\] of collection \[{}\] has no active leader'
                                               .format(SHARD, COLLECTION)):
            self.__controller.verify_shard_health(COLLECTION, SHARD)

    def test_should_raise_exception_if_shard_has_not_enough_active_nodes(self):
        urlopen_mock = MagicMock(side_effect=self.__side_effect_return_cluster_state_only_one_active_replica)
        urllib.request.urlopen = urlopen_mock
        with self.assertRaisesRegex(Exception, 'Shard \[{}\] of collection \[{}\] has not enough active nodes: \[{}\]'
                                               .format(SHARD, COLLECTION, '1')):
            self.__controller.verify_shard_health(COLLECTION, SHARD)

    def __side_effect_all_ok(self, value):
        self.assertIsNotNone(value)
        response_mock = MagicMock()
        response_mock.getcode.return_value = HTTP_CODE_OK
        return response_mock

    def __side_effect_timeout(self, value):
        self.assertIsNotNone(value)
        raise urllib.error.HTTPError(url=None, code=HTTP_CODE_TIMEOUT, msg=None, hdrs=None, fp=None)

    def __side_effect_unknown_http_code(self, value):
        self.assertIsNotNone(value)
        response_mock = MagicMock()
        response_mock.getcode.return_value = HTTP_CODE_UNKNOWN
        return response_mock

    def __side_effect_unknown_http_error(self, value):
        raise urllib.error.HTTPError(url=None, code=HTTP_CODE_UNKNOWN_ERROR, msg=None, hdrs=None, fp=None)

    def __side_effect_error(self, value):
        raise urllib.error.HTTPError(url=None, code=HTTP_CODE_ERROR, msg=None, hdrs=None, fp=None)

    def __side_effect_return_cluster_state(self, value):
        response_mock = MagicMock()
        # return OK for all HTTP requests
        response_mock.getcode.return_value = HTTP_CODE_OK
        response_mock.read.return_value = bytes(json.dumps(CLUSTER_OLD_NODES), 'utf-8')
        return response_mock

    def __side_effect_return_cluster_state_all_registered_nodes(self, value):
        response_mock = MagicMock()
        # return OK for all HTTP requests
        response_mock.getcode.return_value = HTTP_CODE_OK
        response_mock.read.return_value = bytes(json.dumps(CLUSTER_ALL_REGISTERED), 'utf-8')
        return response_mock

    def __side_effect_return_cluster_state_new_nodes(self, value):
        response_mock = MagicMock()
        # return OK for all HTTP requests
        response_mock.getcode.return_value = HTTP_CODE_OK
        response_mock.read.return_value = bytes(json.dumps(CLUSTER_NEW_NODES), 'utf-8')
        return response_mock

    def __side_effect_return_cluster_state_old_nodes(self, value):
        response_mock = MagicMock()
        # return OK for all HTTP requests
        response_mock.getcode.return_value = HTTP_CODE_OK
        response_mock.read.return_value = bytes(json.dumps(CLUSTER_OLD_NODES), 'utf-8')
        return response_mock

    def __side_effect_return_cluster_state_all_nodes(self, value):
        response_mock = MagicMock()
        # return OK for all HTTP requests
        response_mock.getcode.return_value = HTTP_CODE_OK
        response_mock.read.return_value = bytes(json.dumps(CLUSTER_ALL_NODES), 'utf-8')
        return response_mock

    def __side_effect_return_cluster_state_all_nodes_one_not_active(self, value):
        response_mock = MagicMock()
        # return OK for all HTTP requests
        response_mock.getcode.return_value = HTTP_CODE_OK
        response_mock.read.return_value = bytes(json.dumps(CLUSTER_ALL_NODES_ONE_NOT_ACTIVE), 'utf-8')
        return response_mock

    def __side_effect_return_cluster_state_no_leader(self, value):
        regex_api_cluster_status = re.compile('^' + API_URL.replace('.', '\.') + '.*' + '\?action=CLUSTERSTATUS')
        cluster_status_match = regex_api_cluster_status.match(value.get_full_url())
        response_mock = MagicMock()
        # return OK for all HTTP requests
        response_mock.getcode.return_value = HTTP_CODE_OK
        if cluster_status_match:
            response_mock.read.return_value = bytes(json.dumps(CLUSTER_NO_LEADER), 'utf-8')
        return response_mock

    def __side_effect_return_cluster_state_only_one_active_replica(self, value):
        regex_api_cluster_status = re.compile('^' + API_URL.replace('.', '\.') + '.*' + '\?action=CLUSTERSTATUS')
        cluster_status_match = regex_api_cluster_status.match(value.get_full_url())
        response_mock = MagicMock()
        # return OK for all HTTP requests
        response_mock.getcode.return_value = HTTP_CODE_OK
        if cluster_status_match:
            response_mock.read.return_value = bytes(json.dumps(CLUSTER_ONLY_ONE_ACTIVE_REPLICA), 'utf-8')
        return response_mock
Exemple #10
0
def solrcloud_cli(cli_args):
    parser = build_args_parser()
    args = parser.parse_args(cli_args)

    if not args.config:
        args.config = os.path.expanduser(DEFAULT_CONF_FILE)

    if not os.path.exists(args.config):
        print('Configuration file does not exist:', args.config)
        parser.print_usage()
        return

    with open(args.config, 'rb') as fd:
        settings = yaml.load(fd)

    senza_wrapper = SenzaWrapper(args.senza_configuration)
    if args.region:
        senza_wrapper.set_region(args.region)

    for key, value in settings.items():
        senza_wrapper.add_parameter(key, value)

    if args.command in ['bootstrap']:
        controller = ClusterBootstrapController(
            base_url=settings['SolrBaseUrl'],
            stack_name=settings['ApplicationId'],
            sharding_level=args.sharding_level,
            replication_factor=args.replication_level,
            image_version=args.image_version,
            oauth_token=args.token,
            senza_wrapper=senza_wrapper)
    elif args.command in [
            'deploy', 'create-new-cluster', 'delete-old-cluster',
            'add-new-nodes', 'delete-old-nodes', 'switch'
    ]:
        controller = ClusterDeploymentController(
            base_url=settings['SolrBaseUrl'],
            stack_name=settings['ApplicationId'],
            image_version=args.image_version,
            oauth_token=args.token,
            senza_wrapper=senza_wrapper)
    elif args.command in ['delete']:
        controller = ClusterDeleteController(
            base_url=settings['SolrBaseUrl'],
            stack_name=settings['ApplicationId'],
            oauth_token=args.token,
            senza_wrapper=senza_wrapper)
    else:
        print('Unknown command:', args.command)
        parser.print_usage()
        return

    if args.command == 'bootstrap':
        controller.bootstrap_cluster()
    elif args.command == 'deploy':
        controller.deploy_new_version()
    elif args.command == 'delete':
        controller.delete_cluster()
    elif args.command == 'create-new-cluster':
        controller.create_cluster()
    elif args.command == 'delete-old-cluster':
        controller.delete_cluster()
    elif args.command == 'add-new-nodes':
        controller.add_new_nodes_to_cluster()
    elif args.command == 'delete-old-nodes':
        controller.delete_old_nodes_from_cluster()
    elif args.command == 'switch':
        controller.switch_traffic()