def get_sidecar_containers(
self,
system_paasta_config: SystemPaastaConfig) -> List[V1Container]:
registrations = " ".join(self.get_registrations())
# s_m_j currently asserts that services are healthy in smartstack before
# continuing a bounce. this readiness check lets us achieve the same thing
readiness_probe: Optional[V1Probe]
if system_paasta_config.get_enable_nerve_readiness_check():
readiness_probe = V1Probe(
_exec=V1ExecAction(command=[
system_paasta_config.get_nerve_readiness_check_script(),
] + self.get_registrations(), ),
initial_delay_seconds=10,
period_seconds=10,
)
else:
readiness_probe = None
hacheck_sidecar = V1Container(
image=system_paasta_config.get_hacheck_sidecar_image_url(),
lifecycle=V1Lifecycle(pre_stop=V1Handler(_exec=V1ExecAction(
command=[
"/bin/sh",
"-c",
f"/usr/bin/hadown {registrations}; sleep 31",
], ), ), ),
name="hacheck",
env=self.get_kubernetes_environment(),
ports=[
V1ContainerPort(container_port=6666, ),
],
readiness_probe=readiness_probe,
)
return [hacheck_sidecar]
def get_kubernetes_containers(
self,
docker_volumes: Sequence[DockerVolume],
system_paasta_config: SystemPaastaConfig,
aws_ebs_volumes: Sequence[AwsEbsVolume],
service_namespace_config: ServiceNamespaceConfig,
) -> Sequence[V1Container]:
service_container = V1Container(
image=self.get_docker_url(),
command=self.get_cmd(),
args=self.get_args(),
env=self.get_container_env(),
resources=self.get_resource_requirements(),
lifecycle=V1Lifecycle(pre_stop=V1Handler(_exec=V1ExecAction(
command=[
"/bin/sh",
"-c",
"sleep 30",
], ), ), ),
name=self.get_sanitised_deployment_name(),
liveness_probe=self.get_liveness_probe(service_namespace_config),
ports=[
V1ContainerPort(container_port=8888, ),
],
volume_mounts=self.get_volume_mounts(
docker_volumes=docker_volumes,
aws_ebs_volumes=aws_ebs_volumes,
persistent_volumes=self.get_persistent_volumes(),
),
)
containers = [service_container] + self.get_sidecar_containers(
system_paasta_config=system_paasta_config)
return containers
def get_kubernetes_containers(
self,
volumes: Sequence[DockerVolume],
system_paasta_config: SystemPaastaConfig,
) -> Sequence[V1Container]:
service_container = V1Container(
image=self.get_docker_url(),
command=self.get_cmd(),
args=self.get_args(),
env=self.get_container_env(),
lifecycle=V1Lifecycle(
pre_stop=V1Handler(
_exec=V1ExecAction(
command=[
"/bin/sh",
"-c",
"sleep 30",
],
),
),
),
name="{service}-{instance}".format(
service=self.get_sanitised_service_name(),
instance=self.get_sanitised_instance_name(),
),
liveness_probe=V1Probe(
failure_threshold=10,
http_get=V1HTTPGetAction(
path="/status",
port=8888,
),
initial_delay_seconds=15,
period_seconds=10,
timeout_seconds=5,
),
ports=[
V1ContainerPort(
container_port=8888,
),
],
volume_mounts=self.get_volume_mounts(volumes=volumes),
)
containers = [service_container] + self.get_sidecar_containers(system_paasta_config=system_paasta_config)
return containers
def _create_containers(self, service_name: str, deployment_name: str, container_config, mounts,
core_container=False):
cores = container_config.cpu_cores
memory = container_config.ram_mb
min_memory = min(container_config.ram_mb_min, container_config.ram_mb)
environment_variables: list[V1EnvVar] = []
# Use custom health check located in service base
health_probe = V1Probe(
_exec=V1ExecAction(command=["python3", "-m", "assemblyline_v4_service.healthz"]),
timeout_seconds=SERVICE_LIVENESS_TIMEOUT,
period_seconds=SERVICE_LIVENESS_PERIOD)
# If we are launching a core container, include environment variables related to authentication for DBs
if core_container:
environment_variables += [V1EnvVar(name=_n, value=_v) for _n, _v in self.core_env.items()]
environment_variables.append(V1EnvVar(name='PRIVILEGED', value='true'))
# Overwrite them with configured special environment variables
environment_variables += [V1EnvVar(name=_e.name, value=_e.value) for _e in container_config.environment]
# Overwrite those with special hard coded variables
environment_variables += [
V1EnvVar(name='AL_SERVICE_NAME', value=service_name),
V1EnvVar(name='LOG_LEVEL', value=self.log_level)
]
# Overwrite ones defined dynamically by dependency container launches
for name, value in self._service_limited_env[service_name].items():
environment_variables.append(V1EnvVar(name=name, value=value))
image_pull_policy = 'Always' if DEV_MODE else 'IfNotPresent'
return [V1Container(
name=deployment_name,
image=container_config.image,
command=container_config.command,
env=environment_variables,
image_pull_policy=image_pull_policy,
volume_mounts=mounts,
resources=V1ResourceRequirements(
limits={'cpu': cores, 'memory': f'{memory}Mi'},
requests={'cpu': cores*self.cpu_reservation, 'memory': f'{min_memory}Mi'},
),
liveness_probe=health_probe,
readiness_probe=health_probe
)]
def get_liveness_probe(
self,
service_namespace_config: ServiceNamespaceConfig,
) -> Optional[V1Probe]:
mode = self.get_healthcheck_mode(service_namespace_config)
if mode is None:
return None
initial_delay_seconds = self.get_healthcheck_grace_period_seconds()
period_seconds = self.get_healthcheck_interval_seconds()
timeout_seconds = self.get_healthcheck_timeout_seconds()
failure_threshold = self.get_healthcheck_max_consecutive_failures()
probe = V1Probe(
failure_threshold=failure_threshold,
initial_delay_seconds=initial_delay_seconds,
period_seconds=period_seconds,
timeout_seconds=timeout_seconds,
)
if mode == 'http' or mode == 'https':
path = self.get_healthcheck_uri(service_namespace_config)
probe.http_get = V1HTTPGetAction(
path=path,
port=self.get_container_port(),
scheme=mode.upper(),
)
elif mode == 'tcp':
probe.tcp_socket = V1TCPSocketAction(
port=self.get_container_port(), )
elif mode == 'cmd':
probe._exec = V1ExecAction(command=[
"/bin/sh",
"-c",
self.get_healthcheck_cmd(),
], )
else:
raise InvalidHealthcheckMode(
"Unknown mode: %s. Only acceptable healthcheck modes are http/https/tcp"
% mode, )
return probe
def send_create_pod_request(self,
namespace,
name,
image,
args,
ports={},
requests={},
limits={},
probe="",
probe_idelay=3,
probe_period=3,
node_selector=None,
node_name=None,
labels=None):
metadata = V1ObjectMeta(name=name, namespace=namespace, labels=labels)
ports = [
V1ContainerPort(container_port=p, name=n)
for p, n in ports.items()
]
probe_object = None
if probe:
probe_action = V1ExecAction(re.split(r" +", probe))
probe_object = V1Probe(probe_action,
initial_delay_seconds=probe_idelay,
period_seconds=probe_period)
container = V1Container(args=args.split(),
image=image,
name=name,
ports=ports,
resources=V1ResourceRequirements(
requests=requests, limits=limits),
liveness_probe=probe_object)
spec = V1PodSpec(containers=[container],
node_selector=node_selector,
node_name=node_name,
restart_policy="Never")
# {"kubernetes.io/hostname": "10.19.137.148"})
pod = V1Pod(spec=spec, metadata=metadata)
return self.apiV1.create_namespaced_pod(namespace, body=pod)
def add_container(self,
name,
image,
image_pull_policy="IfNotPresent",
args=None,
requests={},
limits={},
probe="",
volumes=[],
ports=[],
**envs):
ports = [p.pod_port for p in ports]
probe_object = None
if probe:
probe_action = V1ExecAction(re.split(r" +", probe))
probe_object = V1Probe(probe_action,
initial_delay_seconds=5,
period_seconds=3)
if args is not None:
args = re.split(r" +", args)
self.volumes.extend(
[v.volume for v in volumes if v.volume not in self.volumes])
volume_mounts = [v.mount for v in volumes]
container_env = [
V1EnvVar(name=k, value=str(v)) for k, v in envs.items()
]
container = V1Container(
args=args,
image=image,
image_pull_policy=image_pull_policy,
name=name,
ports=ports,
resources=V1ResourceRequirements(requests=requests, limits=limits),
liveness_probe=probe_object,
volume_mounts=volume_mounts,
env=container_env,
)
self.containers.append(container)
return self
def test_get_kubernetes_containers(self):
with mock.patch(
'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_docker_url',
autospec=True,
) as mock_get_docker_url, mock.patch(
'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_cmd',
autospec=True,
) as mock_get_cmd, mock.patch(
'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_args',
autospec=True,
) as mock_get_args, mock.patch(
'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_container_env',
autospec=True,
) as mock_get_container_env, mock.patch(
'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_sanitised_service_name',
autospec=True,
return_value='kurupt',
), mock.patch(
'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_sanitised_instance_name',
autospec=True,
return_value='fm',
), mock.patch(
'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_volume_mounts',
autospec=True,
) as mock_get_volume_mounts, mock.patch(
'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_sidecar_containers',
autospec=True,
return_value=['mock_sidecar'],
):
mock_system_config = mock.Mock()
mock_docker_volumes: Sequence[DockerVolume] = []
mock_aws_ebs_volumes: Sequence[AwsEbsVolume] = []
expected = [
V1Container(
args=mock_get_args.return_value,
command=mock_get_cmd.return_value,
env=mock_get_container_env.return_value,
image=mock_get_docker_url.return_value,
lifecycle=V1Lifecycle(pre_stop=V1Handler(
_exec=V1ExecAction(command=[
'/bin/sh',
'-c',
'sleep 30',
], ), ), ),
liveness_probe=V1Probe(
failure_threshold=10,
http_get=V1HTTPGetAction(
path='/status',
port=8888,
),
initial_delay_seconds=15,
period_seconds=10,
timeout_seconds=5,
),
name='kurupt-fm',
ports=[V1ContainerPort(container_port=8888)],
volume_mounts=mock_get_volume_mounts.return_value,
),
'mock_sidecar',
]
assert self.deployment.get_kubernetes_containers(
docker_volumes=mock_docker_volumes,
system_paasta_config=mock_system_config,
aws_ebs_volumes=mock_aws_ebs_volumes,
) == expected
def test_get_sidecar_containers(self):
with mock.patch(
'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_registrations',
autospec=True,
return_value=['universal.credit'],
), mock.patch(
'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_kubernetes_environment',
autospec=True,
return_value={},
), mock.patch(
'paasta_tools.kubernetes_tools.KubernetesDeploymentConfig.get_sanitised_volume_name',
autospec=True,
return_value='sane-name',
):
mock_system_config = mock.Mock(
get_enable_nerve_readiness_check=mock.Mock(return_value=False),
get_nerve_readiness_check_script=mock.Mock(
return_value='/nail/blah.sh'),
get_hacheck_sidecar_image_url=mock.Mock(
return_value='some-docker-image'),
)
ret = self.deployment.get_sidecar_containers(mock_system_config)
expected = [
V1Container(
env={},
image='some-docker-image',
lifecycle=V1Lifecycle(pre_stop=V1Handler(
_exec=V1ExecAction(command=[
'/bin/sh',
'-c',
'/usr/bin/hadown '
'universal.credit; sleep '
'31',
], ), ), ),
name='hacheck',
ports=[V1ContainerPort(container_port=6666)],
),
]
assert ret == expected
mock_system_config = mock.Mock(
get_enable_nerve_readiness_check=mock.Mock(return_value=True),
get_nerve_readiness_check_script=mock.Mock(
return_value='/nail/blah.sh'),
get_hacheck_sidecar_image_url=mock.Mock(
return_value='some-docker-image'),
)
ret = self.deployment.get_sidecar_containers(mock_system_config)
expected = [
V1Container(
env={},
image='some-docker-image',
lifecycle=V1Lifecycle(pre_stop=V1Handler(
_exec=V1ExecAction(command=[
'/bin/sh',
'-c',
'/usr/bin/hadown '
'universal.credit; sleep '
'31',
], ), ), ),
name='hacheck',
ports=[V1ContainerPort(container_port=6666)],
readiness_probe=V1Probe(
_exec=V1ExecAction(
command=['/nail/blah.sh', 'universal.credit'], ),
initial_delay_seconds=10,
period_seconds=10,
),
),
]
assert ret == expected
