示例#1
0
def test_setup_kube_deployment_invalid_job_name():
    with mock.patch(
            "paasta_tools.setup_kubernetes_job.create_application_object",
            autospec=True) as mock_create_application_object, mock.patch(
                "paasta_tools.setup_kubernetes_job.list_all_deployments",
                autospec=True) as mock_list_all_deployments, mock.patch(
                    "paasta_tools.setup_kubernetes_job.log", autospec=True):
        mock_client = mock.Mock()
        mock_list_all_deployments.return_value = [
            KubeDeployment(
                service="kurupt",
                instance="f_m",
                git_sha="",
                image_version=None,
                config_sha="",
                replicas=0,
            )
        ]
        mock_service_instances = ["kuruptf_m"]
        setup_kube_deployments(
            kube_client=mock_client,
            service_instances=mock_service_instances,
            cluster="fake_cluster",
            soa_dir="/nail/blah",
        )
        assert mock_create_application_object.call_count == 0
示例#2
0
def test_list_all_deployments():
    mock_deployments = mock.Mock(items=[])
    mock_client = mock.Mock(deployments=mock.Mock(
        list_namespaced_deployment=mock.Mock(return_value=mock_deployments)))
    assert list_all_deployments(mock_client) == []

    mock_item = mock.Mock(metadata=mock.Mock(labels={
        'service': 'kurupt',
        'instance': 'fm',
        'git_sha': 'a12345',
        'config_sha': 'b12345',
    }, ), )
    type(mock_item).spec = mock.Mock(replicas=3)
    mock_deployments = mock.Mock(items=[mock_item])
    mock_client = mock.Mock(deployments=mock.Mock(
        list_namespaced_deployment=mock.Mock(return_value=mock_deployments)))
    assert list_all_deployments(mock_client) == [
        KubeDeployment(
            service='kurupt',
            instance='fm',
            git_sha='a12345',
            config_sha='b12345',
            replicas=3,
        )
    ]
示例#3
0
    def __init__(
            self,
            item: Union[V1Deployment, V1StatefulSet],
            logging=logging.getLogger(__name__),
    ) -> None:
        """
        This Application wrapper is an interface for creating/deleting k8s deployments and statefulsets
        soa_config is KubernetesDeploymentConfig. It is not loaded in init because it is not always required.
        :param item: Kubernetes Object(V1Deployment/V1StatefulSet) that has already been filled up.
        :param logging: where logs go
        """
        if not item.metadata.namespace:
            item.metadata.namespace = "paasta"
        attrs = {
            attr: item.metadata.labels.get(paasta_prefixed(attr))
            for attr in [
                "service",
                "instance",
                "git_sha",
                "image_version",
                "config_sha",
            ]
        }

        replicas = (item.spec.replicas if item.metadata.labels.get(
            paasta_prefixed("autoscaled"), "false") == "false" else None)
        self.kube_deployment = KubeDeployment(replicas=replicas, **attrs)
        self.item = item
        self.soa_config = None  # type: KubernetesDeploymentConfig
        self.logging = logging
示例#4
0
def reconcile_kubernetes_deployment(
    kube_client: KubeClient,
    service: str,
    instance: str,
    kube_deployments: Sequence[KubeDeployment],
    soa_dir: str,
) -> Tuple[int, Optional[int]]:
    try:
        service_instance_config = load_kubernetes_service_config_no_cache(
            service,
            instance,
            load_system_paasta_config().get_cluster(),
            soa_dir=soa_dir,
        )
    except NoDeploymentsAvailable:
        log.debug("No deployments found for %s.%s in cluster %s. Skipping." %
                  (service, instance, load_system_paasta_config().get_cluster()))
        return 0, None
    except NoConfigurationForServiceError:
        error_msg = "Could not read kubernetes configuration file for %s.%s in cluster %s" % \
                    (service, instance, load_system_paasta_config().get_cluster())
        log.error(error_msg)
        return 1, None

    try:
        formatted_application = service_instance_config.format_kubernetes_app()
    except InvalidKubernetesConfig as e:
        log.error(str(e))
        return (1, None)

    desired_deployment = KubeDeployment(
        service=service,
        instance=instance,
        git_sha=formatted_application.metadata.labels["git_sha"],
        config_sha=formatted_application.metadata.labels["config_sha"],
        replicas=formatted_application.spec.replicas,
    )

    if not (service, instance) in [(kd.service, kd.instance) for kd in kube_deployments]:
        log.debug(f"{desired_deployment} does not exist so creating")
        create_kubernetes_application(
            kube_client=kube_client,
            application=formatted_application,
        )
        return 0, None
    elif desired_deployment not in kube_deployments:
        log.debug(f"{desired_deployment} exists but config_sha or git_sha doesn't match or number of instances changed")
        update_kubernetes_application(
            kube_client=kube_client,
            application=formatted_application,
        )
        return 0, None
    else:
        log.debug(f"{desired_deployment} is up to date, no action taken")
        return 0, None
示例#5
0
 def simple_create_application_object(kube_client, service, instance,
                                      soa_dir):
     fake_app = mock.MagicMock(spec=Application)
     fake_app.kube_deployment = KubeDeployment(service=service,
                                               instance=instance,
                                               git_sha="1",
                                               config_sha="1",
                                               replicas=1)
     fake_app.create = fake_create
     fake_app.update = fake_update
     return True, fake_app
 def simple_create_application_object(
     kube_client, service, instance, cluster, soa_dir
 ):
     fake_app = mock.MagicMock(spec=Application)
     fake_app.kube_deployment = KubeDeployment(
         service=service, instance=instance, git_sha="1", config_sha="1", replicas=1
     )
     fake_app.create = fake_create
     fake_app.update = fake_update
     fake_app.update_related_api_objects = fake_update_related_api_objects
     fake_app.item = None
     fake_app.soa_config = None
     fake_app.__str__ = lambda app: "fake_app"
     return True, fake_app
示例#7
0
 def __init__(
         self,
         item: Union[V1Deployment, V1StatefulSet],
         logging=logging.getLogger(__name__),
 ) -> None:
     """
     This Application wrapper is an interface for creating/deleting k8s deployments and statefulsets
     soa_config is KubernetesDeploymentConfig. It is not loaded in init because it is not always required.
     :param item: Kubernetes Object(V1Deployment/V1StatefulSet) that has already been filled up.
     :param logging: where logs go
     """
     if not item.metadata.namespace:
         item.metadata.namespace = "paasta"
     self.kube_deployment = KubeDeployment(
         service=item.metadata.labels["paasta.yelp.com/service"],
         instance=item.metadata.labels["paasta.yelp.com/instance"],
         git_sha=item.metadata.labels["paasta.yelp.com/git_sha"],
         config_sha=item.metadata.labels["paasta.yelp.com/config_sha"],
         replicas=item.spec.replicas,
     )
     self.item = item
     self.soa_config = None  # type: KubernetesDeploymentConfig
     self.logging = logging
def test_setup_kube_deployment_create_update():
    fake_create = mock.MagicMock()
    fake_update = mock.MagicMock()
    fake_update_related_api_objects = mock.MagicMock()

    def simple_create_application_object(
        kube_client, service, instance, cluster, soa_dir
    ):
        fake_app = mock.MagicMock(spec=Application)
        fake_app.kube_deployment = KubeDeployment(
            service=service, instance=instance, git_sha="1", config_sha="1", replicas=1
        )
        fake_app.create = fake_create
        fake_app.update = fake_update
        fake_app.update_related_api_objects = fake_update_related_api_objects
        fake_app.item = None
        fake_app.soa_config = None
        fake_app.__str__ = lambda app: "fake_app"
        return True, fake_app

    with mock.patch(
        "paasta_tools.setup_kubernetes_job.create_application_object",
        autospec=True,
        side_effect=simple_create_application_object,
    ) as mock_create_application_object, mock.patch(
        "paasta_tools.setup_kubernetes_job.list_all_deployments", autospec=True
    ) as mock_list_all_deployments, mock.patch(
        "paasta_tools.setup_kubernetes_job.log", autospec=True
    ) as mock_log_obj:
        mock_client = mock.Mock()
        # No instances created
        mock_service_instances: Sequence[str] = []
        setup_kube_deployments(
            kube_client=mock_client,
            service_instances=mock_service_instances,
            cluster="fake_cluster",
            soa_dir="/nail/blah",
        )
        assert mock_create_application_object.call_count == 0
        mock_log_obj.info.assert_called_once_with('{"service_instance_updated": []}')
        mock_log_obj.info.reset_mock()

        # Create a new instance
        mock_service_instances = ["kurupt.fm"]
        setup_kube_deployments(
            kube_client=mock_client,
            service_instances=mock_service_instances,
            cluster="fake_cluster",
            soa_dir="/nail/blah",
        )
        assert fake_create.call_count == 1
        assert fake_update.call_count == 0
        mock_log_obj.info.assert_called_with(
            '{"service_instance_updated": ["fake_app"]}'
        )
        mock_log_obj.info.reset_mock()

        # Update when gitsha changed
        fake_create.reset_mock()
        fake_update.reset_mock()
        mock_service_instances = ["kurupt.fm"]
        mock_list_all_deployments.return_value = [
            KubeDeployment(
                service="kurupt", instance="fm", git_sha="2", config_sha="1", replicas=1
            )
        ]
        setup_kube_deployments(
            kube_client=mock_client,
            service_instances=mock_service_instances,
            cluster="fake_cluster",
            soa_dir="/nail/blah",
        )

        assert fake_update.call_count == 1
        assert fake_create.call_count == 0
        mock_log_obj.info.assert_called_with(
            '{"service_instance_updated": ["fake_app"]}'
        )
        mock_log_obj.info.reset_mock()

        # Update when configsha changed
        fake_create.reset_mock()
        fake_update.reset_mock()
        mock_service_instances = ["kurupt.fm"]
        mock_list_all_deployments.return_value = [
            KubeDeployment(
                service="kurupt", instance="fm", git_sha="1", config_sha="2", replicas=1
            )
        ]
        setup_kube_deployments(
            kube_client=mock_client,
            service_instances=mock_service_instances,
            cluster="fake_cluster",
            soa_dir="/nail/blah",
        )
        assert fake_update.call_count == 1
        assert fake_create.call_count == 0
        mock_log_obj.info.assert_called_with(
            '{"service_instance_updated": ["fake_app"]}'
        )
        mock_log_obj.info.reset_mock()

        # Update when replica changed
        fake_create.reset_mock()
        fake_update.reset_mock()
        mock_service_instances = ["kurupt.fm"]
        mock_list_all_deployments.return_value = [
            KubeDeployment(
                service="kurupt", instance="fm", git_sha="1", config_sha="1", replicas=2
            )
        ]
        setup_kube_deployments(
            kube_client=mock_client,
            service_instances=mock_service_instances,
            cluster="fake_cluster",
            soa_dir="/nail/blah",
        )
        assert fake_update.call_count == 1
        assert fake_create.call_count == 0
        mock_log_obj.info.assert_called_with(
            '{"service_instance_updated": ["fake_app"]}'
        )
        mock_log_obj.info.reset_mock()

        # Update one and Create One
        fake_create.reset_mock()
        fake_update.reset_mock()
        mock_service_instances = ["kurupt.fm", "kurupt.garage"]
        mock_list_all_deployments.return_value = [
            KubeDeployment(
                service="kurupt",
                instance="garage",
                git_sha="2",
                config_sha="2",
                replicas=1,
            )
        ]
        setup_kube_deployments(
            kube_client=mock_client,
            service_instances=mock_service_instances,
            cluster="fake_cluster",
            soa_dir="/nail/blah",
        )
        assert fake_update.call_count == 1
        assert fake_create.call_count == 1
        mock_log_obj.info.assert_called_with(
            '{"service_instance_updated": ["fake_app", "fake_app"]}'
        )
        mock_log_obj.info.reset_mock()

        # Always attempt to update related API objects
        fake_create.reset_mock()
        fake_update.reset_mock()
        mock_service_instances = ["kurupt.garage"]
        mock_list_all_deployments.return_value = [
            KubeDeployment(
                service="kurupt",
                instance="garage",
                git_sha="1",
                config_sha="1",
                replicas=1,
            )
        ]
        setup_kube_deployments(
            kube_client=mock_client,
            service_instances=mock_service_instances,
            cluster="fake_cluster",
            soa_dir="/nail/blah",
        )
        assert fake_update.call_count == 0
        assert fake_create.call_count == 0
        assert fake_update_related_api_objects.call_count == 1
        mock_log_obj.info.assert_called_with(
            '{"service_instance_updated": ["fake_app"]}'
        )
示例#9
0
def test_setup_kube_deployment_create_update():
    fake_create = mock.MagicMock()
    fake_update = mock.MagicMock()

    def simple_create_application_object(kube_client, service, instance,
                                         soa_dir):
        fake_app = mock.MagicMock(spec=Application)
        fake_app.kube_deployment = KubeDeployment(service=service,
                                                  instance=instance,
                                                  git_sha="1",
                                                  config_sha="1",
                                                  replicas=1)
        fake_app.create = fake_create
        fake_app.update = fake_update
        return True, fake_app

    with mock.patch(
            "paasta_tools.setup_kubernetes_job.create_application_object",
            autospec=True,
            side_effect=simple_create_application_object,
    ) as mock_create_application_object, mock.patch(
            "paasta_tools.setup_kubernetes_job.list_all_deployments",
            autospec=True) as mock_list_all_deployments:
        mock_client = mock.Mock()
        # No instances created
        mock_service_instances: Sequence[str] = []
        setup_kube_deployments(
            kube_client=mock_client,
            service_instances=mock_service_instances,
            soa_dir="/nail/blah",
        )
        assert mock_create_application_object.call_count == 0

        # Create a new instance
        mock_service_instances = ["kurupt.fm"]
        setup_kube_deployments(
            kube_client=mock_client,
            service_instances=mock_service_instances,
            soa_dir="/nail/blah",
        )
        assert fake_create.call_count == 1
        assert fake_update.call_count == 0

        # Update when gitsha changed
        fake_create.reset_mock()
        fake_update.reset_mock()
        mock_service_instances = ["kurupt.fm"]
        mock_list_all_deployments.return_value = [
            KubeDeployment(service="kurupt",
                           instance="fm",
                           git_sha="2",
                           config_sha="1",
                           replicas=1)
        ]
        setup_kube_deployments(
            kube_client=mock_client,
            service_instances=mock_service_instances,
            soa_dir="/nail/blah",
        )

        assert fake_update.call_count == 1
        assert fake_create.call_count == 0

        # Update when configsha changed
        fake_create.reset_mock()
        fake_update.reset_mock()
        mock_service_instances = ["kurupt.fm"]
        mock_list_all_deployments.return_value = [
            KubeDeployment(service="kurupt",
                           instance="fm",
                           git_sha="1",
                           config_sha="2",
                           replicas=1)
        ]
        setup_kube_deployments(
            kube_client=mock_client,
            service_instances=mock_service_instances,
            soa_dir="/nail/blah",
        )
        assert fake_update.call_count == 1
        assert fake_create.call_count == 0

        # Update when replica changed
        fake_create.reset_mock()
        fake_update.reset_mock()
        mock_service_instances = ["kurupt.fm"]
        mock_list_all_deployments.return_value = [
            KubeDeployment(service="kurupt",
                           instance="fm",
                           git_sha="1",
                           config_sha="1",
                           replicas=2)
        ]
        setup_kube_deployments(
            kube_client=mock_client,
            service_instances=mock_service_instances,
            soa_dir="/nail/blah",
        )
        assert fake_update.call_count == 1
        assert fake_create.call_count == 0

        # Update one and Create One
        fake_create.reset_mock()
        fake_update.reset_mock()
        mock_service_instances = ["kurupt.fm", "kurupt.garage"]
        mock_list_all_deployments.return_value = [
            KubeDeployment(
                service="kurupt",
                instance="garage",
                git_sha="2",
                config_sha="2",
                replicas=1,
            )
        ]
        setup_kube_deployments(
            kube_client=mock_client,
            service_instances=mock_service_instances,
            soa_dir="/nail/blah",
        )
        assert fake_update.call_count == 1
        assert fake_create.call_count == 1

        # not create existing instances
        fake_create.reset_mock()
        fake_update.reset_mock()
        mock_service_instances = ["kurupt.garage"]
        mock_list_all_deployments.return_value = [
            KubeDeployment(
                service="kurupt",
                instance="garage",
                git_sha="1",
                config_sha="1",
                replicas=1,
            )
        ]
        setup_kube_deployments(
            kube_client=mock_client,
            service_instances=mock_service_instances,
            soa_dir="/nail/blah",
        )
        assert fake_update.call_count == 0
        assert fake_create.call_count == 0
示例#10
0
def test_setup_kube_deployment_create_update():
    fake_create = mock.MagicMock()
    fake_update = mock.MagicMock()
    fake_update_related_api_objects = mock.MagicMock()

    def simple_create_application_object(kube_client, service, instance,
                                         cluster, soa_dir):
        fake_app = mock.MagicMock(spec=Application)
        fake_app.kube_deployment = KubeDeployment(
            service=service,
            instance=instance,
            git_sha="1",
            image_version="extrastuff-1",
            config_sha="1",
            replicas=1,
        )
        fake_app.create = fake_create
        fake_app.update = fake_update
        fake_app.update_related_api_objects = fake_update_related_api_objects
        fake_app.item = None
        fake_app.soa_config = None
        fake_app.__str__ = lambda app: "fake_app"
        return True, fake_app

    with mock.patch(
            "paasta_tools.setup_kubernetes_job.create_application_object",
            autospec=True,
            side_effect=simple_create_application_object,
    ) as mock_create_application_object, mock.patch(
            "paasta_tools.setup_kubernetes_job.list_all_deployments",
            autospec=True) as mock_list_all_deployments, mock.patch(
                "paasta_tools.setup_kubernetes_job.log",
                autospec=True) as mock_log_obj, mock.patch(
                    "paasta_tools.setup_kubernetes_job.metrics_lib.NoMetrics",
                    autospec=True) as mock_no_metrics:
        mock_client = mock.Mock()
        # No instances created
        mock_service_instances: Sequence[str] = []
        setup_kube_deployments(
            kube_client=mock_client,
            service_instances=mock_service_instances,
            cluster="fake_cluster",
            soa_dir="/nail/blah",
        )
        assert mock_create_application_object.call_count == 0
        assert fake_update.call_count == 0
        assert fake_update_related_api_objects.call_count == 0
        assert mock_log_obj.info.call_count == 0
        mock_log_obj.info.reset_mock()

        # Create a new instance
        mock_service_instances = ["kurupt.fm"]
        setup_kube_deployments(
            kube_client=mock_client,
            service_instances=mock_service_instances,
            cluster="fake_cluster",
            soa_dir="/nail/blah",
            metrics_interface=mock_no_metrics,
        )
        assert fake_create.call_count == 1
        assert fake_update.call_count == 0
        assert fake_update_related_api_objects.call_count == 1
        assert mock_no_metrics.emit_event.call_count == 1
        mock_log_obj.info.reset_mock()
        mock_no_metrics.reset_mock()

        # Update when gitsha changed
        fake_create.reset_mock()
        fake_update.reset_mock()
        fake_update_related_api_objects.reset_mock()
        mock_service_instances = ["kurupt.fm"]
        mock_list_all_deployments.return_value = [
            KubeDeployment(
                service="kurupt",
                instance="fm",
                git_sha="2",
                image_version="extrastuff-1",
                config_sha="1",
                replicas=1,
            )
        ]
        setup_kube_deployments(
            kube_client=mock_client,
            service_instances=mock_service_instances,
            cluster="fake_cluster",
            soa_dir="/nail/blah",
            metrics_interface=mock_no_metrics,
        )

        assert fake_update.call_count == 1
        assert fake_create.call_count == 0
        assert fake_update_related_api_objects.call_count == 1
        mock_no_metrics.emit_event.assert_called_with(
            name="deploy",
            dimensions={
                "paasta_cluster": "fake_cluster",
                "paasta_service": "kurupt",
                "paasta_instance": "fm",
                "deploy_event": "update",
            },
        )
        mock_log_obj.info.reset_mock()
        mock_no_metrics.reset_mock()

        # Update when image_version changed
        fake_create.reset_mock()
        fake_update.reset_mock()
        fake_update_related_api_objects.reset_mock()
        mock_service_instances = ["kurupt.fm"]
        mock_list_all_deployments.return_value = [
            KubeDeployment(
                service="kurupt",
                instance="fm",
                git_sha="1",
                image_version="extrastuff-2",
                config_sha="1",
                replicas=1,
            )
        ]
        setup_kube_deployments(
            kube_client=mock_client,
            service_instances=mock_service_instances,
            cluster="fake_cluster",
            soa_dir="/nail/blah",
        )
        assert fake_update.call_count == 1
        assert fake_create.call_count == 0
        assert fake_update_related_api_objects.call_count == 1
        mock_log_obj.info.reset_mock()

        # Update when configsha changed
        fake_create.reset_mock()
        fake_update.reset_mock()
        fake_update_related_api_objects.reset_mock()
        mock_service_instances = ["kurupt.fm"]
        mock_list_all_deployments.return_value = [
            KubeDeployment(
                service="kurupt",
                instance="fm",
                git_sha="1",
                image_version="extrastuff-1",
                config_sha="2",
                replicas=1,
            )
        ]
        setup_kube_deployments(
            kube_client=mock_client,
            service_instances=mock_service_instances,
            cluster="fake_cluster",
            soa_dir="/nail/blah",
        )
        assert fake_update.call_count == 1
        assert fake_create.call_count == 0
        assert fake_update_related_api_objects.call_count == 1
        mock_log_obj.info.reset_mock()

        # Update when replica changed
        fake_create.reset_mock()
        fake_update.reset_mock()
        fake_update_related_api_objects.reset_mock()
        mock_service_instances = ["kurupt.fm"]
        mock_list_all_deployments.return_value = [
            KubeDeployment(
                service="kurupt",
                instance="fm",
                git_sha="1",
                image_version="extrastuff-1",
                config_sha="1",
                replicas=2,
            )
        ]
        setup_kube_deployments(
            kube_client=mock_client,
            service_instances=mock_service_instances,
            cluster="fake_cluster",
            soa_dir="/nail/blah",
        )
        assert fake_update.call_count == 1
        assert fake_create.call_count == 0
        assert fake_update_related_api_objects.call_count == 1
        mock_log_obj.info.reset_mock()

        # Update one and Create One
        fake_create.reset_mock()
        fake_update.reset_mock()
        fake_update_related_api_objects.reset_mock()
        mock_service_instances = ["kurupt.fm", "kurupt.garage"]
        mock_list_all_deployments.return_value = [
            KubeDeployment(
                service="kurupt",
                instance="garage",
                git_sha="2",
                image_version="extrastuff-1",
                config_sha="2",
                replicas=1,
            )
        ]
        setup_kube_deployments(
            kube_client=mock_client,
            service_instances=mock_service_instances,
            cluster="fake_cluster",
            soa_dir="/nail/blah",
        )
        assert fake_update.call_count == 1
        assert fake_create.call_count == 1
        assert fake_update_related_api_objects.call_count == 2
        mock_log_obj.info.reset_mock()

        # Always attempt to update related API objects
        fake_create.reset_mock()
        fake_update.reset_mock()
        fake_update_related_api_objects.reset_mock()
        mock_service_instances = ["kurupt.garage"]
        mock_list_all_deployments.return_value = [
            KubeDeployment(
                service="kurupt",
                instance="garage",
                git_sha="1",
                image_version="extrastuff-1",
                config_sha="1",
                replicas=1,
            )
        ]
        setup_kube_deployments(
            kube_client=mock_client,
            service_instances=mock_service_instances,
            cluster="fake_cluster",
            soa_dir="/nail/blah",
        )
        assert fake_update.call_count == 0
        assert fake_create.call_count == 0
        assert fake_update_related_api_objects.call_count == 1
        assert mock_log_obj.info.call_args_list[0] == mock.call(
            "fake_app is up-to-date!")
示例#11
0
def test_reconcile_kubernetes_deployment():
    with mock.patch(
            'paasta_tools.setup_kubernetes_job.load_kubernetes_service_config_no_cache',
            autospec=True,
    ) as mock_load_kubernetes_service_config_no_cache, mock.patch(
            'paasta_tools.setup_kubernetes_job.load_system_paasta_config',
            autospec=True,
    ), mock.patch(
            'paasta_tools.setup_kubernetes_job.create_deployment',
            autospec=True,
    ) as mock_create_deployment, mock.patch(
            'paasta_tools.setup_kubernetes_job.update_deployment',
            autospec=True,
    ) as mock_update_deployment:
        mock_kube_client = mock.Mock()
        mock_deployments: Sequence[KubeDeployment] = []

        # no deployments so should create
        ret = reconcile_kubernetes_deployment(
            kube_client=mock_kube_client,
            service='kurupt',
            instance='fm',
            kube_deployments=mock_deployments,
            soa_dir='/nail/blah',
        )
        assert ret == (0, None)
        mock_deploy = mock_load_kubernetes_service_config_no_cache.return_value.format_kubernetes_app(
        )
        mock_create_deployment.assert_called_with(
            kube_client=mock_kube_client,
            formatted_deployment=mock_deploy,
        )

        # different instance so should create
        mock_deployments = [
            KubeDeployment(
                service='kurupt',
                instance='garage',
                git_sha='a12345',
                config_sha='b12345',
                replicas=3,
            )
        ]
        ret = reconcile_kubernetes_deployment(
            kube_client=mock_kube_client,
            service='kurupt',
            instance='fm',
            kube_deployments=mock_deployments,
            soa_dir='/nail/blah',
        )
        assert ret == (0, None)
        mock_create_deployment.assert_called_with(
            kube_client=mock_kube_client,
            formatted_deployment=mock_deploy,
        )

        # instance correc so do nothing
        mock_create_deployment.reset_mock()
        mock_load_kubernetes_service_config_no_cache.return_value = mock.Mock(
            format_kubernetes_app=mock.Mock(return_value=V1Deployment(
                metadata=V1ObjectMeta(labels={
                    'git_sha': 'a12345',
                    'config_sha': 'b12345',
                }, ),
                spec=V1DeploymentSpec(
                    selector=V1LabelSelector(),
                    template=V1PodTemplateSpec(),
                    replicas=3,
                ),
            ), ))
        mock_deployments = [
            KubeDeployment(
                service='kurupt',
                instance='fm',
                git_sha='a12345',
                config_sha='b12345',
                replicas=3,
            )
        ]
        ret = reconcile_kubernetes_deployment(
            kube_client=mock_kube_client,
            service='kurupt',
            instance='fm',
            kube_deployments=mock_deployments,
            soa_dir='/nail/blah',
        )
        assert ret == (0, None)
        assert not mock_create_deployment.called
        assert not mock_update_deployment.called

        # changed gitsha so update
        mock_create_deployment.reset_mock()
        mock_load_kubernetes_service_config_no_cache.return_value = mock.Mock(
            format_kubernetes_app=mock.Mock(return_value=V1Deployment(
                metadata=V1ObjectMeta(labels={
                    'git_sha': 'new_image',
                    'config_sha': 'b12345',
                }, ),
                spec=V1DeploymentSpec(
                    selector=V1LabelSelector(),
                    template=V1PodTemplateSpec(),
                    replicas=3,
                ),
            ), ))
        mock_deployments = [
            KubeDeployment(
                service='kurupt',
                instance='fm',
                git_sha='a12345',
                config_sha='b12345',
                replicas=3,
            )
        ]
        ret = reconcile_kubernetes_deployment(
            kube_client=mock_kube_client,
            service='kurupt',
            instance='fm',
            kube_deployments=mock_deployments,
            soa_dir='/nail/blah',
        )
        assert ret == (0, None)
        assert not mock_create_deployment.called
        mock_deploy = mock_load_kubernetes_service_config_no_cache.return_value.format_kubernetes_app(
        )
        mock_update_deployment.assert_called_with(
            kube_client=mock_kube_client,
            formatted_deployment=mock_deploy,
        )

        # changed configsha so update
        mock_create_deployment.reset_mock()
        mock_update_deployment.reset_mock()
        mock_load_kubernetes_service_config_no_cache.return_value = mock.Mock(
            format_kubernetes_app=mock.Mock(return_value=V1Deployment(
                metadata=V1ObjectMeta(labels={
                    'git_sha': 'a12345',
                    'config_sha': 'newconfig',
                }, ),
                spec=V1DeploymentSpec(
                    selector=V1LabelSelector(),
                    template=V1PodTemplateSpec(),
                    replicas=3,
                ),
            ), ))
        mock_deployments = [
            KubeDeployment(
                service='kurupt',
                instance='fm',
                git_sha='a12345',
                config_sha='b12345',
                replicas=3,
            )
        ]
        ret = reconcile_kubernetes_deployment(
            kube_client=mock_kube_client,
            service='kurupt',
            instance='fm',
            kube_deployments=mock_deployments,
            soa_dir='/nail/blah',
        )
        assert ret == (0, None)
        assert not mock_create_deployment.called
        mock_deploy = mock_load_kubernetes_service_config_no_cache.return_value.format_kubernetes_app(
        )
        mock_update_deployment.assert_called_with(
            kube_client=mock_kube_client,
            formatted_deployment=mock_deploy,
        )

        # changed number of replicas so update
        mock_create_deployment.reset_mock()
        mock_update_deployment.reset_mock()
        mock_load_kubernetes_service_config_no_cache.return_value = mock.Mock(
            format_kubernetes_app=mock.Mock(return_value=V1Deployment(
                metadata=V1ObjectMeta(labels={
                    'git_sha': 'a12345',
                    'config_sha': 'b12345',
                }, ),
                spec=V1DeploymentSpec(
                    selector=V1LabelSelector(),
                    template=V1PodTemplateSpec(),
                    replicas=2,
                ),
            ), ))
        mock_deployments = [
            KubeDeployment(
                service='kurupt',
                instance='fm',
                git_sha='a12345',
                config_sha='b12345',
                replicas=3,
            )
        ]
        ret = reconcile_kubernetes_deployment(
            kube_client=mock_kube_client,
            service='kurupt',
            instance='fm',
            kube_deployments=mock_deployments,
            soa_dir='/nail/blah',
        )
        assert ret == (0, None)
        assert not mock_create_deployment.called
        mock_deploy = mock_load_kubernetes_service_config_no_cache.return_value.format_kubernetes_app(
        )
        mock_update_deployment.assert_called_with(
            kube_client=mock_kube_client,
            formatted_deployment=mock_deploy,
        )

        # error cases...
        mock_create_deployment.reset_mock()
        mock_update_deployment.reset_mock()
        mock_load_kubernetes_service_config_no_cache.side_effect = NoDeploymentsAvailable
        ret = reconcile_kubernetes_deployment(
            kube_client=mock_kube_client,
            service='kurupt',
            instance='fm',
            kube_deployments=mock_deployments,
            soa_dir='/nail/blah',
        )
        assert ret == (0, None)
        assert not mock_create_deployment.called
        assert not mock_update_deployment.called
        mock_load_kubernetes_service_config_no_cache.side_effect = NoConfigurationForServiceError
        ret = reconcile_kubernetes_deployment(
            kube_client=mock_kube_client,
            service='kurupt',
            instance='fm',
            kube_deployments=mock_deployments,
            soa_dir='/nail/blah',
        )
        assert ret == (1, None)
        assert not mock_create_deployment.called
        assert not mock_update_deployment.called

        mock_load_kubernetes_service_config_no_cache.side_effect = None
        mock_load_kubernetes_service_config_no_cache.return_value = mock.Mock(
            format_kubernetes_app=mock.Mock(side_effect=NoDockerImageError), )
        ret = reconcile_kubernetes_deployment(
            kube_client=mock_kube_client,
            service='kurupt',
            instance='fm',
            kube_deployments=mock_deployments,
            soa_dir='/nail/blah',
        )
        assert ret == (1, None)
        assert not mock_create_deployment.called
        assert not mock_update_deployment.called