def test_invalid_vars_type(self): data = "[1, 2, 3]" with pytest.raises(ParseError) as exc: common.parse_yaml_or_json(data, silent_failure=False) message = str(exc.value) assert "Cannot parse as" in message assert "Input type `list` is not a dictionary" in message
def test_recursive_vars_not_allowed(): rdict = {} rdict['a'] = rdict # YAML dumper will use a tag to give recursive data data = yaml.dump(rdict, default_flow_style=False) with pytest.raises(ParseError) as exc: common.parse_yaml_or_json(data, silent_failure=False) assert 'Circular reference detected' in str(exc)
def test_invalid_JSON_and_YAML(self): data = "{key:val" with pytest.raises(ParseError) as exc: common.parse_yaml_or_json(data, silent_failure=False) message = str(exc.value) assert "Cannot parse as" in message assert self.json_error(data) in message assert self.yaml_error(data) in message
def pod_definition(self): default_pod_spec = { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": settings.AWX_CONTAINER_GROUP_DEFAULT_NAMESPACE }, "spec": { "containers": [{ "image": settings.AWX_CONTAINER_GROUP_DEFAULT_IMAGE, "tty": True, "stdin": True, "imagePullPolicy": "Always", "args": ['sleep', 'infinity'], }] }, } pod_spec_override = {} if self.task and self.task.instance_group.pod_spec_override: pod_spec_override = parse_yaml_or_json( self.task.instance_group.pod_spec_override) pod_spec = {**default_pod_spec, **pod_spec_override} if self.task: pod_spec['metadata'] = deepmerge( pod_spec.get('metadata', {}), dict(name=self.pod_name, labels={ 'ansible-awx': settings.INSTALL_UUID, 'ansible-awx-job-id': str(self.task.id) })) pod_spec['spec']['containers'][0]['name'] = self.pod_name return pod_spec
def pod_definition(self): default_pod_spec = { "apiVersion": "v1", "kind": "Pod", "metadata": { "namespace": settings.AWX_CONTAINER_GROUP_DEFAULT_NAMESPACE }, "spec": { "containers": [{ "image": settings.AWX_CONTAINER_GROUP_DEFAULT_IMAGE, "tty": True, "stdin": True, "imagePullPolicy": "Always", "args": [ 'sleep', 'infinity' ] }] } } pod_spec_override = {} if self.task and self.task.instance_group.pod_spec_override: pod_spec_override = parse_yaml_or_json( self.task.instance_group.pod_spec_override) pod_spec = {**default_pod_spec, **pod_spec_override} if self.task: pod_spec['metadata']['name'] = self.pod_name pod_spec['spec']['containers'][0]['name'] = self.pod_name return pod_spec
def pod_definition(self): ee = self.task.instance.execution_environment default_pod_spec = get_default_pod_spec() pod_spec_override = {} if self.task and self.task.instance.instance_group.pod_spec_override: pod_spec_override = parse_yaml_or_json( self.task.instance.instance_group.pod_spec_override) # According to the deepmerge docstring, the second dictionary will override when # they share keys, which is the desired behavior. # This allows user to only provide elements they want to override, and for us to still provide any # defaults they don't want to change pod_spec = deepmerge(default_pod_spec, pod_spec_override) pod_spec['spec']['containers'][0]['image'] = ee.image pod_spec['spec']['containers'][0]['args'] = [ 'ansible-runner', 'worker', '--private-data-dir=/runner' ] # Enforce EE Pull Policy pull_options = { "always": "Always", "missing": "IfNotPresent", "never": "Never" } if self.task and self.task.instance.execution_environment: if self.task.instance.execution_environment.pull: pod_spec['spec']['containers'][0][ 'imagePullPolicy'] = pull_options[ self.task.instance.execution_environment.pull] if self.task and self.task.instance.is_container_group_task: # If EE credential is passed, create an imagePullSecret if self.task.instance.execution_environment and self.task.instance.execution_environment.credential: # Create pull secret in k8s cluster based on ee cred from awx.main.scheduler.kubernetes import PodManager # prevent circular import pm = PodManager(self.task.instance) secret_name = pm.create_secret(job=self.task.instance) # Inject secret name into podspec pod_spec['spec']['imagePullSecrets'] = [{"name": secret_name}] if self.task: pod_spec['metadata'] = deepmerge( pod_spec.get('metadata', {}), dict(name=self.pod_name, labels={ 'ansible-awx': settings.INSTALL_UUID, 'ansible-awx-job-id': str(self.task.instance.id) }), ) return pod_spec
def back_out_new_instance_id(apps, source, new_id): Host = apps.get_model('main', 'Host') modified_ct = 0 for host in Host.objects.filter(inventory_sources__source=source).iterator(): host_vars = parse_yaml_or_json(host.variables) predicted_id_value = _get_instance_id(host_vars, new_id) if predicted_id_value != host.instance_id: logger.debug('Host {}-{} did not get its instance_id from {}, skipping'.format(host.name, host.pk, new_id)) continue host.instance_id = '' host.save(update_fields=['instance_id']) modified_ct += 1 if modified_ct: logger.info('Reverse migrated instance ID for {} hosts imported by {} source'.format(modified_ct, source))
def pod_definition(self): default_pod_spec = get_default_pod_spec() pod_spec_override = {} if self.task and self.task.instance_group.pod_spec_override: pod_spec_override = parse_yaml_or_json(self.task.instance_group.pod_spec_override) pod_spec = {**default_pod_spec, **pod_spec_override} if self.task: pod_spec['metadata'] = deepmerge( pod_spec.get('metadata', {}), dict(name=self.pod_name, labels={'ansible-awx': settings.INSTALL_UUID, 'ansible-awx-job-id': str(self.task.id)}) ) pod_spec['spec']['containers'][0]['name'] = self.pod_name return pod_spec
def _get_instance_id_for_upgrade(host, new_id): if host.instance_id: # this should not have happened, but nothing to really do about it... logger.debug('Unexpectedly, host {}-{} has instance_id set'.format(host.name, host.pk)) return None host_vars = parse_yaml_or_json(host.variables) new_id_value = _get_instance_id(host_vars, new_id) if not new_id_value: # another source type with overwrite_vars or pesky users could have done this logger.info('Host {}-{} has no {} var, probably due to separate modifications'.format(host.name, host.pk, new_id)) return None if len(new_id) > 255: # this should never happen logger.warn('Computed instance id "{}"" for host {}-{} is too long'.format(new_id_value, host.name, host.pk)) return None return new_id_value
def test_parse_yaml_or_json(input_, output): assert common.parse_yaml_or_json(input_) == output
def pod_definition(self): ee = self.task.instance.execution_environment default_pod_spec = get_default_pod_spec() pod_spec_override = {} if self.task and self.task.instance.instance_group.pod_spec_override: pod_spec_override = parse_yaml_or_json( self.task.instance.instance_group.pod_spec_override) # According to the deepmerge docstring, the second dictionary will override when # they share keys, which is the desired behavior. # This allows user to only provide elements they want to override, and for us to still provide any # defaults they don't want to change pod_spec = deepmerge(default_pod_spec, pod_spec_override) pod_spec['spec']['containers'][0]['image'] = ee.image pod_spec['spec']['containers'][0]['args'] = [ 'ansible-runner', 'worker', '--private-data-dir=/runner' ] # Enforce EE Pull Policy pull_options = { "always": "Always", "missing": "IfNotPresent", "never": "Never" } if self.task and self.task.instance.execution_environment: if self.task.instance.execution_environment.pull: pod_spec['spec']['containers'][0][ 'imagePullPolicy'] = pull_options[ self.task.instance.execution_environment.pull] # This allows the user to also expose the isolated path list # to EEs running in k8s/ocp environments, i.e. container groups. # This assumes the node and SA supports hostPath volumes # type is not passed due to backward compatibility, # which means that no checks will be performed before mounting the hostPath volume. if settings.AWX_MOUNT_ISOLATED_PATHS_ON_K8S and settings.AWX_ISOLATION_SHOW_PATHS: spec_volume_mounts = [] spec_volumes = [] for idx, this_path in enumerate(settings.AWX_ISOLATION_SHOW_PATHS): mount_option = None if this_path.count(':') == MAX_ISOLATED_PATH_COLON_DELIMITER: src, dest, mount_option = this_path.split(':') elif this_path.count( ':') == MAX_ISOLATED_PATH_COLON_DELIMITER - 1: src, dest = this_path.split(':') else: src = dest = this_path # Enforce read-only volume if 'ro' has been explicitly passed # We do this so we can use the same configuration for regular scenarios and k8s # Since flags like ':O', ':z' or ':Z' are not valid in the k8s realm # Example: /data:/data:ro read_only = bool('ro' == mount_option) # Since type is not being passed, k8s by default will not perform any checks if the # hostPath volume exists on the k8s node itself. spec_volumes.append({ 'name': f'volume-{idx}', 'hostPath': { 'path': src } }) spec_volume_mounts.append({ 'name': f'volume-{idx}', 'mountPath': f'{dest}', 'readOnly': read_only }) # merge any volumes definition already present in the pod_spec if 'volumes' in pod_spec['spec']: pod_spec['spec']['volumes'] += spec_volumes else: pod_spec['spec']['volumes'] = spec_volumes # merge any volumesMounts definition already present in the pod_spec if 'volumeMounts' in pod_spec['spec']['containers'][0]: pod_spec['spec']['containers'][0][ 'volumeMounts'] += spec_volume_mounts else: pod_spec['spec']['containers'][0][ 'volumeMounts'] = spec_volume_mounts if self.task and self.task.instance.is_container_group_task: # If EE credential is passed, create an imagePullSecret if self.task.instance.execution_environment and self.task.instance.execution_environment.credential: # Create pull secret in k8s cluster based on ee cred from awx.main.scheduler.kubernetes import PodManager # prevent circular import pm = PodManager(self.task.instance) secret_name = pm.create_secret(job=self.task.instance) # Inject secret name into podspec pod_spec['spec']['imagePullSecrets'] = [{"name": secret_name}] if self.task: pod_spec['metadata'] = deepmerge( pod_spec.get('metadata', {}), dict(name=self.pod_name, labels={ 'ansible-awx': settings.INSTALL_UUID, 'ansible-awx-job-id': str(self.task.instance.id) }), ) return pod_spec