def check_kubernetes_pod_replication( instance_config: KubernetesDeploymentConfig, all_tasks_or_pods: Sequence[V1Pod], smartstack_replication_checker: KubeSmartstackReplicationChecker, ) -> Optional[bool]: """Checks a service's replication levels based on how the service's replication should be monitored. (smartstack or k8s) :param instance_config: an instance of KubernetesDeploymentConfig :param smartstack_replication_checker: an instance of KubeSmartstackReplicationChecker """ expected_count = instance_config.get_instances() log.info("Expecting %d total tasks for %s" % (expected_count, instance_config.job_id)) proxy_port = get_proxy_port_for_instance(instance_config) registrations = instance_config.get_registrations() # if the primary registration does not match the service_instance name then # the best we can do is check k8s for replication (for now). if proxy_port is not None and registrations[0] == instance_config.job_id: is_well_replicated = monitoring_tools.check_smartstack_replication_for_instance( instance_config=instance_config, expected_count=expected_count, smartstack_replication_checker=smartstack_replication_checker, ) return is_well_replicated else: check_healthy_kubernetes_tasks_for_service_instance( instance_config=instance_config, expected_count=expected_count, all_pods=all_tasks_or_pods, ) return None
def kubernetes_smartstack_status( service: str, instance: str, job_config: kubernetes_tools.KubernetesDeploymentConfig, service_namespace_config: ServiceNamespaceConfig, pods: Sequence[V1Pod], should_return_individual_backends: bool = False, ) -> Mapping[str, Any]: registration = job_config.get_registrations()[0] instance_pool = job_config.get_pool() smartstack_replication_checker = KubeSmartstackReplicationChecker( nodes=kubernetes_tools.get_all_nodes(settings.kubernetes_client), system_paasta_config=settings.system_paasta_config, ) node_hostname_by_location = smartstack_replication_checker.get_allowed_locations_and_hosts( job_config) expected_smartstack_count = marathon_tools.get_expected_instance_count_for_namespace( service=service, namespace=instance, cluster=settings.cluster, instance_type_class=KubernetesDeploymentConfig, ) expected_count_per_location = int(expected_smartstack_count / len(node_hostname_by_location)) smartstack_status: MutableMapping[str, Any] = { "registration": registration, "expected_backends_per_location": expected_count_per_location, "locations": [], } for location, hosts in node_hostname_by_location.items(): synapse_host = smartstack_replication_checker.get_first_host_in_pool( hosts, instance_pool) sorted_backends = sorted( get_backends( registration, synapse_host=synapse_host, synapse_port=settings.system_paasta_config.get_synapse_port(), synapse_haproxy_url_format=settings.system_paasta_config. get_synapse_haproxy_url_format(), ), key=lambda backend: backend["status"], reverse=True, # put 'UP' backends above 'MAINT' backends ) matched_backends_and_pods = match_backends_and_pods( sorted_backends, pods) location_dict = build_smartstack_location_dict( location, matched_backends_and_pods, should_return_individual_backends) smartstack_status["locations"].append(location_dict) return smartstack_status
def setUp(self): mock_config_dict = KubernetesDeploymentConfigDict( bounce_method='crossover', ) self.deployment = KubernetesDeploymentConfig( service='kurupt', instance='fm', cluster='brentford', config_dict=mock_config_dict, branch_dict=None, soa_dir='/nail/blah', )
def kubernetes_job_status( kstatus: MutableMapping[str, Any], client: kubernetes_tools.KubeClient, job_config: kubernetes_tools.KubernetesDeploymentConfig, pod_list: Sequence[V1Pod], replicaset_list: Sequence[V1ReplicaSet], verbose: int, ) -> None: app_id = job_config.get_sanitised_deployment_name() kstatus["app_id"] = app_id kstatus["pods"] = [] kstatus["replicasets"] = [] if verbose > 0: for pod in pod_list: kstatus["pods"].append({ "name": pod.metadata.name, "host": pod.spec.node_name, "deployed_timestamp": pod.metadata.creation_timestamp.timestamp(), "phase": pod.status.phase, }) for replicaset in replicaset_list: kstatus["replicasets"].append({ "name": replicaset.metadata.name, "replicas": replicaset.spec.replicas, "ready_replicas": replicaset.status.ready_replicas, "create_timestamp": replicaset.metadata.creation_timestamp.timestamp(), }) kstatus["expected_instance_count"] = job_config.get_instances() app = kubernetes_tools.get_kubernetes_app_by_name(app_id, client) deploy_status = kubernetes_tools.get_kubernetes_app_deploy_status( client, app, job_config.get_instances()) kstatus[ "deploy_status"] = kubernetes_tools.KubernetesDeployStatus.tostring( deploy_status) kstatus["running_instance_count"] = (app.status.ready_replicas if app.status.ready_replicas else 0) kstatus["create_timestamp"] = app.metadata.creation_timestamp.timestamp() kstatus["namespace"] = app.metadata.namespace
def kubernetes_job_status( kstatus: MutableMapping[str, Any], client: kubernetes_tools.KubeClient, job_config: kubernetes_tools.KubernetesDeploymentConfig, pod_list: Sequence[V1Pod], verbose: bool, ) -> None: app_id = job_config.get_sanitised_deployment_name() kstatus['app_id'] = app_id if verbose is True: kstatus['slaves'] = [ pod.spec.node_name for pod in pod_list ] kstatus['expected_instance_count'] = job_config.get_instances() app = kubernetes_tools.get_kubernetes_app_by_name(app_id, client) deploy_status = kubernetes_tools.get_kubernetes_app_deploy_status(client, app, job_config.get_instances()) kstatus['deploy_status'] = kubernetes_tools.KubernetesDeployStatus.tostring(deploy_status) kstatus['running_instance_count'] = app.status.ready_replicas if app.status.ready_replicas else 0
def setup_app(config_dict, exists_hpa): item = mock.MagicMock() item.metadata.name = "fake_name" item.metadata.namespace = "faasta" app = DeploymentWrapper(item=item) app.soa_config = KubernetesDeploymentConfig( service="service", cluster="cluster", instance="instance", config_dict=config_dict, branch_dict=None, ) app.exists_hpa = mock.Mock(return_value=exists_hpa) app.delete_horizontal_pod_autoscaler = mock.Mock(return_value=None) return app
def check_kubernetes_pod_replication( instance_config: KubernetesDeploymentConfig, all_tasks_or_pods: Sequence[V1Pod], smartstack_replication_checker: KubeSmartstackReplicationChecker, default_alert_after: Optional[str] = DEFAULT_ALERT_AFTER, ) -> Optional[bool]: """Checks a service's replication levels based on how the service's replication should be monitored. (smartstack or k8s) :param instance_config: an instance of KubernetesDeploymentConfig :param smartstack_replication_checker: an instance of KubeSmartstackReplicationChecker """ expected_count = instance_config.get_instances() log.info( "Expecting %d total tasks for %s" % (expected_count, instance_config.job_id) ) proxy_port = get_proxy_port_for_instance(instance_config) registrations = instance_config.get_registrations() # If this instance does not autoscale and only has 1 instance, set alert after to 20m. # Otherwise, set it to 10 min. if ( not instance_config.is_autoscaling_enabled() and instance_config.get_instances() == 1 ): default_alert_after = "20m" if "monitoring" not in instance_config.config_dict: instance_config.config_dict["monitoring"] = {} instance_config.config_dict["monitoring"][ "alert_after" ] = instance_config.config_dict["monitoring"].get( "alert_after", default_alert_after ) # if the primary registration does not match the service_instance name then # the best we can do is check k8s for replication (for now). if proxy_port is not None and registrations[0] == instance_config.job_id: is_well_replicated = monitoring_tools.check_smartstack_replication_for_instance( instance_config=instance_config, expected_count=expected_count, smartstack_replication_checker=smartstack_replication_checker, ) return is_well_replicated else: check_healthy_kubernetes_tasks_for_service_instance( instance_config=instance_config, expected_count=expected_count, all_pods=all_tasks_or_pods, ) return None
class TestKubernetesDeploymentConfig(unittest.TestCase): def setUp(self): mock_config_dict = KubernetesDeploymentConfigDict( bounce_method='crossover', ) self.deployment = KubernetesDeploymentConfig( service='kurupt', instance='fm', cluster='brentford', config_dict=mock_config_dict, branch_dict=None, soa_dir='/nail/blah', ) def test_copy(self): assert self.deployment.copy() == self.deployment assert self.deployment.copy() is not self.deployment def test_get_bounce_method(self): assert self.deployment.get_bounce_method() == 'RollingUpdate' self.deployment.config_dict['bounce_method'] = 'downthenup' assert self.deployment.get_bounce_method() == 'Recreate' def test_get_deployment_strategy(self): with mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_bounce_method', autospec=True, return_value='RollingUpdate', ) as mock_get_bounce_method: assert self.deployment.get_deployment_strategy_config( ) == V1DeploymentStrategy( type='RollingUpdate', rolling_update=V1RollingUpdateDeployment( max_surge='100%', max_unavailable='0%', ), ) mock_get_bounce_method.return_value = 'Recreate' assert self.deployment.get_deployment_strategy_config( ) == V1DeploymentStrategy(type='Recreate', ) def test_get_sanitised_volume_name(self): self.deployment.get_sanitised_volume_name( '/var/tmp') == 'slash-varslash-tmp' self.deployment.get_sanitised_volume_name( '/var/tmp/') == 'slash-varslash-tmp' def test_get_sidecar_containers(self): with mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_registrations', autospec=True, return_value=['universal.credit'], ), mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_kubernetes_environment', autospec=True, return_value={}, ), mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_sanitised_volume_name', autospec=True, return_value='sane-name', ): mock_system_config = mock.Mock( get_enable_nerve_readiness_check=mock.Mock(return_value=False), get_nerve_readiness_check_script=mock.Mock( return_value='/nail/blah.sh'), get_hacheck_sidecar_image_url=mock.Mock( return_value='some-docker-image'), ) ret = self.deployment.get_sidecar_containers(mock_system_config) expected = [ V1Container( env={}, image='some-docker-image', lifecycle=V1Lifecycle(pre_stop=V1Handler( _exec=V1ExecAction(command=[ '/bin/sh', '-c', '/usr/bin/hadown ' 'universal.credit; sleep ' '31', ], ), ), ), name='hacheck', ports=[V1ContainerPort(container_port=6666)], ), ] assert ret == expected mock_system_config = mock.Mock( get_enable_nerve_readiness_check=mock.Mock(return_value=True), get_nerve_readiness_check_script=mock.Mock( return_value='/nail/blah.sh'), get_hacheck_sidecar_image_url=mock.Mock( return_value='some-docker-image'), ) ret = self.deployment.get_sidecar_containers(mock_system_config) expected = [ V1Container( env={}, image='some-docker-image', lifecycle=V1Lifecycle(pre_stop=V1Handler( _exec=V1ExecAction(command=[ '/bin/sh', '-c', '/usr/bin/hadown ' 'universal.credit; sleep ' '31', ], ), ), ), name='hacheck', ports=[V1ContainerPort(container_port=6666)], readiness_probe=V1Probe( _exec=V1ExecAction( command=['/nail/blah.sh', 'universal.credit'], ), initial_delay_seconds=10, period_seconds=10, ), ), ] assert ret == expected def test_get_container_env(self): with mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_env', autospec=True, return_value={ 'mc': 'grindah', 'dj': 'beats', }, ), mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_kubernetes_environment', autospec=True, return_value=[ V1EnvVar( name='manager', value='chabuddy', ), ], ): expected = [ V1EnvVar(name='mc', value='grindah'), V1EnvVar(name='dj', value='beats'), V1EnvVar(name='manager', value='chabuddy'), ] assert expected == self.deployment.get_container_env() def test_get_kubernetes_environment(self): ret = self.deployment.get_kubernetes_environment() assert 'PAASTA_POD_IP' in [env.name for env in ret] def test_get_kubernetes_containers(self): with mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_docker_url', autospec=True, ) as mock_get_docker_url, mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_cmd', autospec=True, ) as mock_get_cmd, mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_args', autospec=True, ) as mock_get_args, mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_container_env', autospec=True, ) as mock_get_container_env, mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_sanitised_service_name', autospec=True, return_value='kurupt', ), mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_sanitised_instance_name', autospec=True, return_value='fm', ), mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_volume_mounts', autospec=True, ) as mock_get_volume_mounts, mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_sidecar_containers', autospec=True, return_value=['mock_sidecar'], ): mock_system_config = mock.Mock() mock_docker_volumes: Sequence[DockerVolume] = [] mock_aws_ebs_volumes: Sequence[AwsEbsVolume] = [] expected = [ V1Container( args=mock_get_args.return_value, command=mock_get_cmd.return_value, env=mock_get_container_env.return_value, image=mock_get_docker_url.return_value, lifecycle=V1Lifecycle(pre_stop=V1Handler( _exec=V1ExecAction(command=[ '/bin/sh', '-c', 'sleep 30', ], ), ), ), liveness_probe=V1Probe( failure_threshold=10, http_get=V1HTTPGetAction( path='/status', port=8888, ), initial_delay_seconds=15, period_seconds=10, timeout_seconds=5, ), name='kurupt-fm', ports=[V1ContainerPort(container_port=8888)], volume_mounts=mock_get_volume_mounts.return_value, ), 'mock_sidecar', ] assert self.deployment.get_kubernetes_containers( docker_volumes=mock_docker_volumes, system_paasta_config=mock_system_config, aws_ebs_volumes=mock_aws_ebs_volumes, ) == expected def test_get_pod_volumes(self): mock_docker_volumes = [ { 'hostPath': '/nail/blah', 'containerPath': '/nail/foo' }, { 'hostPath': '/nail/thing', 'containerPath': '/nail/bar' }, ] mock_aws_ebs_volumes = [ { 'volume_id': 'vol-ZZZZZZZZZZZZZZZZZ', 'fs_type': 'ext4', 'container_path': '/nail/qux' }, ] expected_volumes = [ V1Volume( host_path=V1HostPathVolumeSource(path='/nail/blah', ), name='host--slash-nailslash-blah', ), V1Volume( host_path=V1HostPathVolumeSource(path='/nail/thing', ), name='host--slash-nailslash-thing', ), V1Volume( aws_elastic_block_store=V1AWSElasticBlockStoreVolumeSource( volume_id='vol-ZZZZZZZZZZZZZZZZZ', fs_type='ext4', read_only=False, ), name='aws-ebs--vol-ZZZZZZZZZZZZZZZZZ', ), ] assert self.deployment.get_pod_volumes( docker_volumes=mock_docker_volumes, aws_ebs_volumes=mock_aws_ebs_volumes, ) == expected_volumes def test_get_volume_mounts(self): with mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_sanitised_volume_name', autospec=True, return_value='some-volume', ): mock_docker_volumes = [ { 'hostPath': '/nail/blah', 'containerPath': '/nail/foo' }, { 'hostPath': '/nail/thing', 'containerPath': '/nail/bar', 'mode': 'RW' }, ] mock_aws_ebs_volumes = [ { 'volume_id': 'vol-ZZZZZZZZZZZZZZZZZ', 'fs_type': 'ext4', 'container_path': '/nail/qux' }, ] expected_volumes = [ V1VolumeMount( mount_path='/nail/foo', name='some-volume', read_only=True, ), V1VolumeMount( mount_path='/nail/bar', name='some-volume', read_only=False, ), V1VolumeMount( mount_path='/nail/qux', name='some-volume', read_only=True, ), ] assert self.deployment.get_volume_mounts( docker_volumes=mock_docker_volumes, aws_ebs_volumes=mock_aws_ebs_volumes, ) == expected_volumes def test_get_sanitised_service_name(self): with mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_service', autospec=True, return_value='my_service', ): assert self.deployment.get_sanitised_service_name( ) == 'my--service' def test_get_sanitised_instance_name(self): with mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_instance', autospec=True, return_value='my_instance', ): assert self.deployment.get_sanitised_instance_name( ) == 'my--instance' def test_format_kubernetes_app_dict(self): with mock.patch( 'paasta_tools.kubernetes_tools.load_system_paasta_config', autospec=True, ) as mock_load_system_config, mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_docker_url', autospec=True, ) as mock_get_docker_url, mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_volumes', autospec=True, ) as mock_get_volumes, mock.patch( 'paasta_tools.kubernetes_tools.get_code_sha_from_dockerurl', autospec=True, ) as mock_get_code_sha_from_dockerurl, mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_sanitised_service_name', autospec=True, return_value='kurupt', ), mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_sanitised_instance_name', autospec=True, return_value='fm', ), mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_service', autospec=True, ) as mock_get_service, mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_instance', autospec=True, ) as mock_get_instance, mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_instances', autospec=True, ) as mock_get_instances, mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_deployment_strategy_config', autospec=True, ) as mock_get_deployment_strategy_config, mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_kubernetes_containers', autospec=True, ) as mock_get_kubernetes_containers, mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_pod_volumes', autospec=True, return_value=[], ) as mock_get_pod_volumes, mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_sanitised_volume_name', autospec=True, ), mock.patch( 'paasta_tools.kubernetes_tools.get_config_hash', autospec=True, ) as mock_get_config_hash, mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_force_bounce', autospec=True, ) as mock_get_force_bounce, mock.patch( 'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.sanitize_for_config_hash', autospec=True, ) as mock_sanitize_for_config_hash: ret = self.deployment.format_kubernetes_app() assert mock_load_system_config.called assert mock_get_docker_url.called assert mock_get_volumes.called assert mock_get_pod_volumes.called mock_get_config_hash.assert_called_with( mock_sanitize_for_config_hash.return_value, force_bounce=mock_get_force_bounce.return_value, ) expected = V1Deployment( metadata=V1ObjectMeta( labels={ 'config_sha': mock_get_config_hash.return_value, 'git_sha': mock_get_code_sha_from_dockerurl.return_value, 'instance': mock_get_instance.return_value, 'service': mock_get_service.return_value, }, name='kurupt-fm', ), spec=V1DeploymentSpec( replicas=mock_get_instances.return_value, selector=V1LabelSelector(match_labels={ 'instance': mock_get_instance.return_value, 'service': mock_get_service.return_value, }, ), strategy=mock_get_deployment_strategy_config.return_value, template=V1PodTemplateSpec( metadata=V1ObjectMeta(labels={ 'config_sha': mock_get_config_hash.return_value, 'git_sha': mock_get_code_sha_from_dockerurl.return_value, 'instance': mock_get_instance.return_value, 'service': mock_get_service.return_value, }, ), spec=V1PodSpec( containers=mock_get_kubernetes_containers. return_value, restart_policy='Always', volumes=[], ), ), ), ) assert ret == expected def test_sanitize_config_hash(self): mock_config = V1Deployment( metadata=V1ObjectMeta( name='qwe', labels={ 'mc': 'grindah', }, ), spec=V1DeploymentSpec( replicas=2, selector=V1LabelSelector(match_labels={ 'freq': '108.9', }, ), template=V1PodTemplateSpec(), ), ) ret = self.deployment.sanitize_for_config_hash(mock_config) assert 'replicas' not in ret['spec'].keys() def test_get_bounce_margin_factor(self): assert isinstance(self.deployment.get_bounce_margin_factor(), float)
def test_sync_boto_secrets(): with mock.patch( "paasta_tools.kubernetes.bin.paasta_secrets_sync.open", create=True, autospec=None, ) as mock_open, mock.patch( "paasta_tools.kubernetes.bin.paasta_secrets_sync.get_kubernetes_secret_signature", autospec=True, ) as mock_get_kubernetes_secret_signature, mock.patch( "paasta_tools.kubernetes.bin.paasta_secrets_sync.create_plaintext_dict_secret", autospec=True, ) as mock_create_secret, mock.patch( "paasta_tools.kubernetes.bin.paasta_secrets_sync.create_kubernetes_secret_signature", autospec=True, ) as mock_create_kubernetes_secret_signature, mock.patch( "paasta_tools.kubernetes.bin.paasta_secrets_sync.update_plaintext_dict_secret", autospec=True, ) as mock_update_secret, mock.patch( "paasta_tools.kubernetes.bin.paasta_secrets_sync.update_kubernetes_secret_signature", autospec=True, ) as mock_update_kubernetes_secret_signature, mock.patch( "paasta_tools.kubernetes.bin.paasta_secrets_sync.PaastaServiceConfigLoader", autospec=True, ) as mock_config_loader, mock.patch( "paasta_tools.kubernetes.bin.paasta_secrets_sync.time.sleep", autospec=True, ): mock_client = mock.Mock() config_dict = {"boto_keys": ["scribereader"]} deployment = KubernetesDeploymentConfig( service="my-service", instance="my-instance", cluster="mega-cluster", config_dict=config_dict, branch_dict=None, soa_dir="/nail/blah", ) mock_loader = mock.MagicMock() mock_loader.instance_configs.return_value = [deployment] mock_config_loader.return_value = mock_loader mock_open.return_value = mock.MagicMock() mock_handle = mock_open.return_value.__enter__.return_value expected_secret_data = { "scribereader-sh": "ZmlsZTE=", "scribereader-yaml": "ZmlsZTI=", "scribereader-json": "ZmlsZTM=", "scribereader-cfg": "ZmlsZTQ=", } expected_signature = "4c3da4da5d97294f69527dc92c2b930ce127522c" # New secret mock_handle.read.side_effect = ["file1", "file2", "file3", "file4"] mock_get_kubernetes_secret_signature.return_value = None assert sync_boto_secrets( kube_client=mock_client, cluster="westeros-prod", service="universe", secret_provider_name="vaulty", vault_cluster_config={}, soa_dir="/nail/blah", namespace="paasta", ) assert mock_create_secret.called assert not mock_update_secret.called call_args = mock_create_secret.call_args_list assert call_args[0][1]["secret_data"] == expected_secret_data assert mock_create_kubernetes_secret_signature.called # Update secret mock_handle.read.side_effect = ["file1", "file2", "file3", "file4"] mock_get_kubernetes_secret_signature.return_value = "1235abc" assert sync_boto_secrets( kube_client=mock_client, cluster="westeros-prod", service="universe", secret_provider_name="vaulty", vault_cluster_config={}, soa_dir="/nail/blah", namespace="paasta", ) assert mock_update_secret.called call_args = mock_update_secret.call_args_list assert call_args[0][1]["secret_data"] == expected_secret_data assert mock_update_kubernetes_secret_signature.called # No changes needed mock_handle.read.side_effect = ["file1", "file2", "file3", "file4"] mock_get_kubernetes_secret_signature.return_value = expected_signature mock_update_secret.reset_mock() mock_create_secret.reset_mock() assert sync_boto_secrets( kube_client=mock_client, cluster="westeros-prod", service="universe", secret_provider_name="vaulty", vault_cluster_config={}, soa_dir="/nail/blah", namespace="paasta", ) assert not mock_update_secret.called assert not mock_create_secret.called # No signature, but secret exists mock_update_secret.reset_mock() mock_create_secret.reset_mock() mock_handle.read.side_effect = ["file1", "file2", "file3", "file4"] mock_get_kubernetes_secret_signature.return_value = None mock_create_secret.side_effect = ApiException(409) mock_update_kubernetes_secret_signature.reset_mock() assert sync_boto_secrets( kube_client=mock_client, cluster="westeros-prod", service="universe", secret_provider_name="vaulty", vault_cluster_config={}, soa_dir="/nail/blah", namespace="paasta", ) assert mock_get_kubernetes_secret_signature.called assert mock_create_secret.called assert mock_update_secret.called assert mock_create_kubernetes_secret_signature.called assert not mock_update_kubernetes_secret_signature.called