def awx_k8s_reaper(): if not settings.RECEPTOR_RELEASE_WORK: return from awx.main.scheduler.kubernetes import PodManager # prevent circular import for group in InstanceGroup.objects.filter( is_container_group=True).iterator(): logger.debug("Checking for orphaned k8s pods for {}.".format(group)) pods = PodManager.list_active_jobs(group) for job in UnifiedJob.objects.filter(pk__in=pods.keys()).exclude( status__in=ACTIVE_STATES): logger.debug( '{} is no longer active, reaping orphaned k8s pod'.format( job.log_format)) try: pm = PodManager(job) pm.kube_api.delete_namespaced_pod( name=pods[job.id], namespace=pm.namespace, _request_timeout=settings. AWX_CONTAINER_GROUP_K8S_API_TIMEOUT) except Exception: logger.exception( "Failed to delete orphaned pod {} from {}".format( job.log_format, group))
def test_pod_manager_namespace_property(job): pm = PodManager(job) assert pm.namespace == settings.AWX_CONTAINER_GROUP_DEFAULT_NAMESPACE job.instance_group.pod_spec_override = """ metadata: namespace: my-namespace """ assert PodManager(job).namespace == 'my-namespace'
def pod_definition(self): ee = self.task.instance.execution_environment default_pod_spec = get_default_pod_spec() pod_spec_override = {} if self.task and self.task.instance.instance_group.pod_spec_override: pod_spec_override = parse_yaml_or_json( self.task.instance.instance_group.pod_spec_override) # According to the deepmerge docstring, the second dictionary will override when # they share keys, which is the desired behavior. # This allows user to only provide elements they want to override, and for us to still provide any # defaults they don't want to change pod_spec = deepmerge(default_pod_spec, pod_spec_override) pod_spec['spec']['containers'][0]['image'] = ee.image pod_spec['spec']['containers'][0]['args'] = [ 'ansible-runner', 'worker', '--private-data-dir=/runner' ] # Enforce EE Pull Policy pull_options = { "always": "Always", "missing": "IfNotPresent", "never": "Never" } if self.task and self.task.instance.execution_environment: if self.task.instance.execution_environment.pull: pod_spec['spec']['containers'][0][ 'imagePullPolicy'] = pull_options[ self.task.instance.execution_environment.pull] if self.task and self.task.instance.is_container_group_task: # If EE credential is passed, create an imagePullSecret if self.task.instance.execution_environment and self.task.instance.execution_environment.credential: # Create pull secret in k8s cluster based on ee cred from awx.main.scheduler.kubernetes import PodManager # prevent circular import pm = PodManager(self.task.instance) secret_name = pm.create_secret(job=self.task.instance) # Inject secret name into podspec pod_spec['spec']['imagePullSecrets'] = [{"name": secret_name}] if self.task: pod_spec['metadata'] = deepmerge( pod_spec.get('metadata', {}), dict(name=self.pod_name, labels={ 'ansible-awx': settings.INSTALL_UUID, 'ansible-awx-job-id': str(self.task.instance.id) }), ) return pod_spec
def determine_actions(self, request, view): # Add field information for GET requests (so field names/labels are # available even when we can't POST/PUT). actions = {} for method in {'GET', 'PUT', 'POST'} & set(view.allowed_methods): view.request = clone_request(request, method) obj = None try: # Test global permissions if hasattr(view, 'check_permissions'): view.check_permissions(view.request) # Test object permissions if method == 'PUT' and hasattr(view, 'get_object'): obj = view.get_object() except (exceptions.APIException, PermissionDenied, Http404): continue else: # If user has appropriate permissions for the view, include # appropriate metadata about the fields that should be supplied. serializer = view.get_serializer(instance=obj) actions[method] = self.get_serializer_info(serializer, method=method) finally: view.request = request for field, meta in list(actions[method].items()): if not isinstance(meta, dict): continue if field == "pod_spec_override": meta['default'] = PodManager().pod_definition # Add type choices if available from the serializer. if field == 'type' and hasattr(serializer, 'get_type_choices'): meta['choices'] = serializer.get_type_choices() # For GET method, remove meta attributes that aren't relevant # when reading a field and remove write-only fields. if method == 'GET': attrs_to_remove = ('required', 'read_only', 'default', 'min_length', 'max_length', 'placeholder') for attr in attrs_to_remove: meta.pop(attr, None) meta.get('child', {}).pop(attr, None) if meta.pop('write_only', False): actions['GET'].pop(field) # For PUT/POST methods, remove read-only fields. if method in ('PUT', 'POST'): # This value should always be False for PUT/POST, so don't # show it (file-based read-only settings can't be updated) meta.pop('defined_in_file', False) if meta.pop('read_only', False): if field == 'id' and hasattr(view, 'attach'): continue actions[method].pop(field) return actions
def test_custom_pod_spec(job): job.instance_group.pod_spec_override = """ spec: containers: - image: my-custom-image """ custom_image = PodManager(job).pod_definition['spec']['containers'][0]['image'] assert custom_image == 'my-custom-image'
def test_kubectl_ssl_verification(containerized_job): cred = containerized_job.instance_group.credential cred.inputs['verify_ssl'] = True key_material = subprocess.run('openssl genrsa 2> /dev/null', shell=True, check=True, stdout=subprocess.PIPE) key = create_temporary_fifo(key_material.stdout) cmd = f""" openssl req -x509 -sha256 -new -nodes \ -key {key} -subj '/C=US/ST=North Carolina/L=Durham/O=Ansible/OU=AWX Development/CN=awx.localhost' """ cert = subprocess.run(cmd.strip(), shell=True, check=True, stdout=subprocess.PIPE) cred.inputs['ssl_ca_cert'] = cert.stdout cred.save() pm = PodManager(containerized_job) ca_data = pm.kube_config['clusters'][0]['cluster']['certificate-authority-data'] assert cert.stdout == base64.b64decode(ca_data.encode())
def pod_definition(self): ee = self.task.instance.execution_environment default_pod_spec = get_default_pod_spec() pod_spec_override = {} if self.task and self.task.instance.instance_group.pod_spec_override: pod_spec_override = parse_yaml_or_json( self.task.instance.instance_group.pod_spec_override) # According to the deepmerge docstring, the second dictionary will override when # they share keys, which is the desired behavior. # This allows user to only provide elements they want to override, and for us to still provide any # defaults they don't want to change pod_spec = deepmerge(default_pod_spec, pod_spec_override) pod_spec['spec']['containers'][0]['image'] = ee.image pod_spec['spec']['containers'][0]['args'] = [ 'ansible-runner', 'worker', '--private-data-dir=/runner' ] # Enforce EE Pull Policy pull_options = { "always": "Always", "missing": "IfNotPresent", "never": "Never" } if self.task and self.task.instance.execution_environment: if self.task.instance.execution_environment.pull: pod_spec['spec']['containers'][0][ 'imagePullPolicy'] = pull_options[ self.task.instance.execution_environment.pull] # This allows the user to also expose the isolated path list # to EEs running in k8s/ocp environments, i.e. container groups. # This assumes the node and SA supports hostPath volumes # type is not passed due to backward compatibility, # which means that no checks will be performed before mounting the hostPath volume. if settings.AWX_MOUNT_ISOLATED_PATHS_ON_K8S and settings.AWX_ISOLATION_SHOW_PATHS: spec_volume_mounts = [] spec_volumes = [] for idx, this_path in enumerate(settings.AWX_ISOLATION_SHOW_PATHS): mount_option = None if this_path.count(':') == MAX_ISOLATED_PATH_COLON_DELIMITER: src, dest, mount_option = this_path.split(':') elif this_path.count( ':') == MAX_ISOLATED_PATH_COLON_DELIMITER - 1: src, dest = this_path.split(':') else: src = dest = this_path # Enforce read-only volume if 'ro' has been explicitly passed # We do this so we can use the same configuration for regular scenarios and k8s # Since flags like ':O', ':z' or ':Z' are not valid in the k8s realm # Example: /data:/data:ro read_only = bool('ro' == mount_option) # Since type is not being passed, k8s by default will not perform any checks if the # hostPath volume exists on the k8s node itself. spec_volumes.append({ 'name': f'volume-{idx}', 'hostPath': { 'path': src } }) spec_volume_mounts.append({ 'name': f'volume-{idx}', 'mountPath': f'{dest}', 'readOnly': read_only }) # merge any volumes definition already present in the pod_spec if 'volumes' in pod_spec['spec']: pod_spec['spec']['volumes'] += spec_volumes else: pod_spec['spec']['volumes'] = spec_volumes # merge any volumesMounts definition already present in the pod_spec if 'volumeMounts' in pod_spec['spec']['containers'][0]: pod_spec['spec']['containers'][0][ 'volumeMounts'] += spec_volume_mounts else: pod_spec['spec']['containers'][0][ 'volumeMounts'] = spec_volume_mounts if self.task and self.task.instance.is_container_group_task: # If EE credential is passed, create an imagePullSecret if self.task.instance.execution_environment and self.task.instance.execution_environment.credential: # Create pull secret in k8s cluster based on ee cred from awx.main.scheduler.kubernetes import PodManager # prevent circular import pm = PodManager(self.task.instance) secret_name = pm.create_secret(job=self.task.instance) # Inject secret name into podspec pod_spec['spec']['imagePullSecrets'] = [{"name": secret_name}] if self.task: pod_spec['metadata'] = deepmerge( pod_spec.get('metadata', {}), dict(name=self.pod_name, labels={ 'ansible-awx': settings.INSTALL_UUID, 'ansible-awx-job-id': str(self.task.instance.id) }), ) return pod_spec
def test_default_pod_spec(job): default_image = PodManager(job).pod_definition['spec']['containers'][0]['image'] assert default_image == settings.AWX_CONTAINER_GROUP_DEFAULT_IMAGE