def awx_k8s_reaper():
if not settings.RECEPTOR_RELEASE_WORK:
return
from awx.main.scheduler.kubernetes import PodManager # prevent circular import
for group in InstanceGroup.objects.filter(
is_container_group=True).iterator():
logger.debug("Checking for orphaned k8s pods for {}.".format(group))
pods = PodManager.list_active_jobs(group)
for job in UnifiedJob.objects.filter(pk__in=pods.keys()).exclude(
status__in=ACTIVE_STATES):
logger.debug(
'{} is no longer active, reaping orphaned k8s pod'.format(
job.log_format))
try:
pm = PodManager(job)
pm.kube_api.delete_namespaced_pod(
name=pods[job.id],
namespace=pm.namespace,
_request_timeout=settings.
AWX_CONTAINER_GROUP_K8S_API_TIMEOUT)
except Exception:
logger.exception(
"Failed to delete orphaned pod {} from {}".format(
job.log_format, group))
def test_pod_manager_namespace_property(job):
pm = PodManager(job)
assert pm.namespace == settings.AWX_CONTAINER_GROUP_DEFAULT_NAMESPACE
job.instance_group.pod_spec_override = """
metadata:
namespace: my-namespace
"""
assert PodManager(job).namespace == 'my-namespace'
def pod_definition(self):
ee = self.task.instance.execution_environment
default_pod_spec = get_default_pod_spec()
pod_spec_override = {}
if self.task and self.task.instance.instance_group.pod_spec_override:
pod_spec_override = parse_yaml_or_json(
self.task.instance.instance_group.pod_spec_override)
# According to the deepmerge docstring, the second dictionary will override when
# they share keys, which is the desired behavior.
# This allows user to only provide elements they want to override, and for us to still provide any
# defaults they don't want to change
pod_spec = deepmerge(default_pod_spec, pod_spec_override)
pod_spec['spec']['containers'][0]['image'] = ee.image
pod_spec['spec']['containers'][0]['args'] = [
'ansible-runner', 'worker', '--private-data-dir=/runner'
]
# Enforce EE Pull Policy
pull_options = {
"always": "Always",
"missing": "IfNotPresent",
"never": "Never"
}
if self.task and self.task.instance.execution_environment:
if self.task.instance.execution_environment.pull:
pod_spec['spec']['containers'][0][
'imagePullPolicy'] = pull_options[
self.task.instance.execution_environment.pull]
if self.task and self.task.instance.is_container_group_task:
# If EE credential is passed, create an imagePullSecret
if self.task.instance.execution_environment and self.task.instance.execution_environment.credential:
# Create pull secret in k8s cluster based on ee cred
from awx.main.scheduler.kubernetes import PodManager # prevent circular import
pm = PodManager(self.task.instance)
secret_name = pm.create_secret(job=self.task.instance)
# Inject secret name into podspec
pod_spec['spec']['imagePullSecrets'] = [{"name": secret_name}]
if self.task:
pod_spec['metadata'] = deepmerge(
pod_spec.get('metadata', {}),
dict(name=self.pod_name,
labels={
'ansible-awx': settings.INSTALL_UUID,
'ansible-awx-job-id': str(self.task.instance.id)
}),
)
return pod_spec
def determine_actions(self, request, view):
# Add field information for GET requests (so field names/labels are
# available even when we can't POST/PUT).
actions = {}
for method in {'GET', 'PUT', 'POST'} & set(view.allowed_methods):
view.request = clone_request(request, method)
obj = None
try:
# Test global permissions
if hasattr(view, 'check_permissions'):
view.check_permissions(view.request)
# Test object permissions
if method == 'PUT' and hasattr(view, 'get_object'):
obj = view.get_object()
except (exceptions.APIException, PermissionDenied, Http404):
continue
else:
# If user has appropriate permissions for the view, include
# appropriate metadata about the fields that should be supplied.
serializer = view.get_serializer(instance=obj)
actions[method] = self.get_serializer_info(serializer,
method=method)
finally:
view.request = request
for field, meta in list(actions[method].items()):
if not isinstance(meta, dict):
continue
if field == "pod_spec_override":
meta['default'] = PodManager().pod_definition
# Add type choices if available from the serializer.
if field == 'type' and hasattr(serializer, 'get_type_choices'):
meta['choices'] = serializer.get_type_choices()
# For GET method, remove meta attributes that aren't relevant
# when reading a field and remove write-only fields.
if method == 'GET':
attrs_to_remove = ('required', 'read_only', 'default',
'min_length', 'max_length',
'placeholder')
for attr in attrs_to_remove:
meta.pop(attr, None)
meta.get('child', {}).pop(attr, None)
if meta.pop('write_only', False):
actions['GET'].pop(field)
# For PUT/POST methods, remove read-only fields.
if method in ('PUT', 'POST'):
# This value should always be False for PUT/POST, so don't
# show it (file-based read-only settings can't be updated)
meta.pop('defined_in_file', False)
if meta.pop('read_only', False):
if field == 'id' and hasattr(view, 'attach'):
continue
actions[method].pop(field)
return actions
def test_custom_pod_spec(job):
job.instance_group.pod_spec_override = """
spec:
containers:
- image: my-custom-image
"""
custom_image = PodManager(job).pod_definition['spec']['containers'][0]['image']
assert custom_image == 'my-custom-image'
def test_kubectl_ssl_verification(containerized_job):
cred = containerized_job.instance_group.credential
cred.inputs['verify_ssl'] = True
key_material = subprocess.run('openssl genrsa 2> /dev/null',
shell=True, check=True,
stdout=subprocess.PIPE)
key = create_temporary_fifo(key_material.stdout)
cmd = f"""
openssl req -x509 -sha256 -new -nodes \
-key {key} -subj '/C=US/ST=North Carolina/L=Durham/O=Ansible/OU=AWX Development/CN=awx.localhost'
"""
cert = subprocess.run(cmd.strip(), shell=True, check=True, stdout=subprocess.PIPE)
cred.inputs['ssl_ca_cert'] = cert.stdout
cred.save()
pm = PodManager(containerized_job)
ca_data = pm.kube_config['clusters'][0]['cluster']['certificate-authority-data']
assert cert.stdout == base64.b64decode(ca_data.encode())
def pod_definition(self):
ee = self.task.instance.execution_environment
default_pod_spec = get_default_pod_spec()
pod_spec_override = {}
if self.task and self.task.instance.instance_group.pod_spec_override:
pod_spec_override = parse_yaml_or_json(
self.task.instance.instance_group.pod_spec_override)
# According to the deepmerge docstring, the second dictionary will override when
# they share keys, which is the desired behavior.
# This allows user to only provide elements they want to override, and for us to still provide any
# defaults they don't want to change
pod_spec = deepmerge(default_pod_spec, pod_spec_override)
pod_spec['spec']['containers'][0]['image'] = ee.image
pod_spec['spec']['containers'][0]['args'] = [
'ansible-runner', 'worker', '--private-data-dir=/runner'
]
# Enforce EE Pull Policy
pull_options = {
"always": "Always",
"missing": "IfNotPresent",
"never": "Never"
}
if self.task and self.task.instance.execution_environment:
if self.task.instance.execution_environment.pull:
pod_spec['spec']['containers'][0][
'imagePullPolicy'] = pull_options[
self.task.instance.execution_environment.pull]
# This allows the user to also expose the isolated path list
# to EEs running in k8s/ocp environments, i.e. container groups.
# This assumes the node and SA supports hostPath volumes
# type is not passed due to backward compatibility,
# which means that no checks will be performed before mounting the hostPath volume.
if settings.AWX_MOUNT_ISOLATED_PATHS_ON_K8S and settings.AWX_ISOLATION_SHOW_PATHS:
spec_volume_mounts = []
spec_volumes = []
for idx, this_path in enumerate(settings.AWX_ISOLATION_SHOW_PATHS):
mount_option = None
if this_path.count(':') == MAX_ISOLATED_PATH_COLON_DELIMITER:
src, dest, mount_option = this_path.split(':')
elif this_path.count(
':') == MAX_ISOLATED_PATH_COLON_DELIMITER - 1:
src, dest = this_path.split(':')
else:
src = dest = this_path
# Enforce read-only volume if 'ro' has been explicitly passed
# We do this so we can use the same configuration for regular scenarios and k8s
# Since flags like ':O', ':z' or ':Z' are not valid in the k8s realm
# Example: /data:/data:ro
read_only = bool('ro' == mount_option)
# Since type is not being passed, k8s by default will not perform any checks if the
# hostPath volume exists on the k8s node itself.
spec_volumes.append({
'name': f'volume-{idx}',
'hostPath': {
'path': src
}
})
spec_volume_mounts.append({
'name': f'volume-{idx}',
'mountPath': f'{dest}',
'readOnly': read_only
})
# merge any volumes definition already present in the pod_spec
if 'volumes' in pod_spec['spec']:
pod_spec['spec']['volumes'] += spec_volumes
else:
pod_spec['spec']['volumes'] = spec_volumes
# merge any volumesMounts definition already present in the pod_spec
if 'volumeMounts' in pod_spec['spec']['containers'][0]:
pod_spec['spec']['containers'][0][
'volumeMounts'] += spec_volume_mounts
else:
pod_spec['spec']['containers'][0][
'volumeMounts'] = spec_volume_mounts
if self.task and self.task.instance.is_container_group_task:
# If EE credential is passed, create an imagePullSecret
if self.task.instance.execution_environment and self.task.instance.execution_environment.credential:
# Create pull secret in k8s cluster based on ee cred
from awx.main.scheduler.kubernetes import PodManager # prevent circular import
pm = PodManager(self.task.instance)
secret_name = pm.create_secret(job=self.task.instance)
# Inject secret name into podspec
pod_spec['spec']['imagePullSecrets'] = [{"name": secret_name}]
if self.task:
pod_spec['metadata'] = deepmerge(
pod_spec.get('metadata', {}),
dict(name=self.pod_name,
labels={
'ansible-awx': settings.INSTALL_UUID,
'ansible-awx-job-id': str(self.task.instance.id)
}),
)
return pod_spec
def test_default_pod_spec(job):
default_image = PodManager(job).pod_definition['spec']['containers'][0]['image']
assert default_image == settings.AWX_CONTAINER_GROUP_DEFAULT_IMAGE