def test_invalid_vars_type(self):
data = "[1, 2, 3]"
with pytest.raises(ParseError) as exc:
common.parse_yaml_or_json(data, silent_failure=False)
message = str(exc.value)
assert "Cannot parse as" in message
assert "Input type `list` is not a dictionary" in message
def test_recursive_vars_not_allowed():
rdict = {}
rdict['a'] = rdict
# YAML dumper will use a tag to give recursive data
data = yaml.dump(rdict, default_flow_style=False)
with pytest.raises(ParseError) as exc:
common.parse_yaml_or_json(data, silent_failure=False)
assert 'Circular reference detected' in str(exc)
def test_invalid_JSON_and_YAML(self):
data = "{key:val"
with pytest.raises(ParseError) as exc:
common.parse_yaml_or_json(data, silent_failure=False)
message = str(exc.value)
assert "Cannot parse as" in message
assert self.json_error(data) in message
assert self.yaml_error(data) in message
def pod_definition(self):
default_pod_spec = {
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
"namespace": settings.AWX_CONTAINER_GROUP_DEFAULT_NAMESPACE
},
"spec": {
"containers": [{
"image": settings.AWX_CONTAINER_GROUP_DEFAULT_IMAGE,
"tty": True,
"stdin": True,
"imagePullPolicy": "Always",
"args": ['sleep', 'infinity'],
}]
},
}
pod_spec_override = {}
if self.task and self.task.instance_group.pod_spec_override:
pod_spec_override = parse_yaml_or_json(
self.task.instance_group.pod_spec_override)
pod_spec = {**default_pod_spec, **pod_spec_override}
if self.task:
pod_spec['metadata'] = deepmerge(
pod_spec.get('metadata', {}),
dict(name=self.pod_name,
labels={
'ansible-awx': settings.INSTALL_UUID,
'ansible-awx-job-id': str(self.task.id)
}))
pod_spec['spec']['containers'][0]['name'] = self.pod_name
return pod_spec
def pod_definition(self):
default_pod_spec = {
"apiVersion": "v1",
"kind": "Pod",
"metadata": {
"namespace": settings.AWX_CONTAINER_GROUP_DEFAULT_NAMESPACE
},
"spec": {
"containers": [{
"image": settings.AWX_CONTAINER_GROUP_DEFAULT_IMAGE,
"tty": True,
"stdin": True,
"imagePullPolicy": "Always",
"args": [
'sleep', 'infinity'
]
}]
}
}
pod_spec_override = {}
if self.task and self.task.instance_group.pod_spec_override:
pod_spec_override = parse_yaml_or_json(
self.task.instance_group.pod_spec_override)
pod_spec = {**default_pod_spec, **pod_spec_override}
if self.task:
pod_spec['metadata']['name'] = self.pod_name
pod_spec['spec']['containers'][0]['name'] = self.pod_name
return pod_spec
def pod_definition(self):
ee = self.task.instance.execution_environment
default_pod_spec = get_default_pod_spec()
pod_spec_override = {}
if self.task and self.task.instance.instance_group.pod_spec_override:
pod_spec_override = parse_yaml_or_json(
self.task.instance.instance_group.pod_spec_override)
# According to the deepmerge docstring, the second dictionary will override when
# they share keys, which is the desired behavior.
# This allows user to only provide elements they want to override, and for us to still provide any
# defaults they don't want to change
pod_spec = deepmerge(default_pod_spec, pod_spec_override)
pod_spec['spec']['containers'][0]['image'] = ee.image
pod_spec['spec']['containers'][0]['args'] = [
'ansible-runner', 'worker', '--private-data-dir=/runner'
]
# Enforce EE Pull Policy
pull_options = {
"always": "Always",
"missing": "IfNotPresent",
"never": "Never"
}
if self.task and self.task.instance.execution_environment:
if self.task.instance.execution_environment.pull:
pod_spec['spec']['containers'][0][
'imagePullPolicy'] = pull_options[
self.task.instance.execution_environment.pull]
if self.task and self.task.instance.is_container_group_task:
# If EE credential is passed, create an imagePullSecret
if self.task.instance.execution_environment and self.task.instance.execution_environment.credential:
# Create pull secret in k8s cluster based on ee cred
from awx.main.scheduler.kubernetes import PodManager # prevent circular import
pm = PodManager(self.task.instance)
secret_name = pm.create_secret(job=self.task.instance)
# Inject secret name into podspec
pod_spec['spec']['imagePullSecrets'] = [{"name": secret_name}]
if self.task:
pod_spec['metadata'] = deepmerge(
pod_spec.get('metadata', {}),
dict(name=self.pod_name,
labels={
'ansible-awx': settings.INSTALL_UUID,
'ansible-awx-job-id': str(self.task.instance.id)
}),
)
return pod_spec
def back_out_new_instance_id(apps, source, new_id):
Host = apps.get_model('main', 'Host')
modified_ct = 0
for host in Host.objects.filter(inventory_sources__source=source).iterator():
host_vars = parse_yaml_or_json(host.variables)
predicted_id_value = _get_instance_id(host_vars, new_id)
if predicted_id_value != host.instance_id:
logger.debug('Host {}-{} did not get its instance_id from {}, skipping'.format(host.name, host.pk, new_id))
continue
host.instance_id = ''
host.save(update_fields=['instance_id'])
modified_ct += 1
if modified_ct:
logger.info('Reverse migrated instance ID for {} hosts imported by {} source'.format(modified_ct, source))
def pod_definition(self):
default_pod_spec = get_default_pod_spec()
pod_spec_override = {}
if self.task and self.task.instance_group.pod_spec_override:
pod_spec_override = parse_yaml_or_json(self.task.instance_group.pod_spec_override)
pod_spec = {**default_pod_spec, **pod_spec_override}
if self.task:
pod_spec['metadata'] = deepmerge(
pod_spec.get('metadata', {}), dict(name=self.pod_name, labels={'ansible-awx': settings.INSTALL_UUID, 'ansible-awx-job-id': str(self.task.id)})
)
pod_spec['spec']['containers'][0]['name'] = self.pod_name
return pod_spec
def _get_instance_id_for_upgrade(host, new_id):
if host.instance_id:
# this should not have happened, but nothing to really do about it...
logger.debug('Unexpectedly, host {}-{} has instance_id set'.format(host.name, host.pk))
return None
host_vars = parse_yaml_or_json(host.variables)
new_id_value = _get_instance_id(host_vars, new_id)
if not new_id_value:
# another source type with overwrite_vars or pesky users could have done this
logger.info('Host {}-{} has no {} var, probably due to separate modifications'.format(host.name, host.pk, new_id))
return None
if len(new_id) > 255:
# this should never happen
logger.warn('Computed instance id "{}"" for host {}-{} is too long'.format(new_id_value, host.name, host.pk))
return None
return new_id_value
def test_parse_yaml_or_json(input_, output):
assert common.parse_yaml_or_json(input_) == output
def pod_definition(self):
ee = self.task.instance.execution_environment
default_pod_spec = get_default_pod_spec()
pod_spec_override = {}
if self.task and self.task.instance.instance_group.pod_spec_override:
pod_spec_override = parse_yaml_or_json(
self.task.instance.instance_group.pod_spec_override)
# According to the deepmerge docstring, the second dictionary will override when
# they share keys, which is the desired behavior.
# This allows user to only provide elements they want to override, and for us to still provide any
# defaults they don't want to change
pod_spec = deepmerge(default_pod_spec, pod_spec_override)
pod_spec['spec']['containers'][0]['image'] = ee.image
pod_spec['spec']['containers'][0]['args'] = [
'ansible-runner', 'worker', '--private-data-dir=/runner'
]
# Enforce EE Pull Policy
pull_options = {
"always": "Always",
"missing": "IfNotPresent",
"never": "Never"
}
if self.task and self.task.instance.execution_environment:
if self.task.instance.execution_environment.pull:
pod_spec['spec']['containers'][0][
'imagePullPolicy'] = pull_options[
self.task.instance.execution_environment.pull]
# This allows the user to also expose the isolated path list
# to EEs running in k8s/ocp environments, i.e. container groups.
# This assumes the node and SA supports hostPath volumes
# type is not passed due to backward compatibility,
# which means that no checks will be performed before mounting the hostPath volume.
if settings.AWX_MOUNT_ISOLATED_PATHS_ON_K8S and settings.AWX_ISOLATION_SHOW_PATHS:
spec_volume_mounts = []
spec_volumes = []
for idx, this_path in enumerate(settings.AWX_ISOLATION_SHOW_PATHS):
mount_option = None
if this_path.count(':') == MAX_ISOLATED_PATH_COLON_DELIMITER:
src, dest, mount_option = this_path.split(':')
elif this_path.count(
':') == MAX_ISOLATED_PATH_COLON_DELIMITER - 1:
src, dest = this_path.split(':')
else:
src = dest = this_path
# Enforce read-only volume if 'ro' has been explicitly passed
# We do this so we can use the same configuration for regular scenarios and k8s
# Since flags like ':O', ':z' or ':Z' are not valid in the k8s realm
# Example: /data:/data:ro
read_only = bool('ro' == mount_option)
# Since type is not being passed, k8s by default will not perform any checks if the
# hostPath volume exists on the k8s node itself.
spec_volumes.append({
'name': f'volume-{idx}',
'hostPath': {
'path': src
}
})
spec_volume_mounts.append({
'name': f'volume-{idx}',
'mountPath': f'{dest}',
'readOnly': read_only
})
# merge any volumes definition already present in the pod_spec
if 'volumes' in pod_spec['spec']:
pod_spec['spec']['volumes'] += spec_volumes
else:
pod_spec['spec']['volumes'] = spec_volumes
# merge any volumesMounts definition already present in the pod_spec
if 'volumeMounts' in pod_spec['spec']['containers'][0]:
pod_spec['spec']['containers'][0][
'volumeMounts'] += spec_volume_mounts
else:
pod_spec['spec']['containers'][0][
'volumeMounts'] = spec_volume_mounts
if self.task and self.task.instance.is_container_group_task:
# If EE credential is passed, create an imagePullSecret
if self.task.instance.execution_environment and self.task.instance.execution_environment.credential:
# Create pull secret in k8s cluster based on ee cred
from awx.main.scheduler.kubernetes import PodManager # prevent circular import
pm = PodManager(self.task.instance)
secret_name = pm.create_secret(job=self.task.instance)
# Inject secret name into podspec
pod_spec['spec']['imagePullSecrets'] = [{"name": secret_name}]
if self.task:
pod_spec['metadata'] = deepmerge(
pod_spec.get('metadata', {}),
dict(name=self.pod_name,
labels={
'ansible-awx': settings.INSTALL_UUID,
'ansible-awx-job-id': str(self.task.instance.id)
}),
)
return pod_spec