def test_adjust_specific_container():
dep = """
apiVersion: apps/v1
kind: Deployment
metadata:
name: test-adjust-specific-container
spec:
selector:
matchLabels:
app: test-adjust-specific-container
template:
metadata:
labels:
app: test-adjust-specific-container
spec:
containers:
- name: main
image: alpine:latest
command: ["/bin/sh", "-c", "sleep 3600"]
env:
- name: ANYTHING
value: everything
- name: collateral
image: alpine:latest
command: ["/bin/sh", "-c", "sleep 3600"]
env:
- name: ANYTHING
value: nothing
"""
cfg = """
k8s:
application:
components:
test-adjust-specific-container/collateral:
env:
ANYTHING:
values:
- everything
- nothing
"""
setup_deployment(dep)
adjust_dep(
cfg, {
'application': {
'components': {
'test-adjust-specific-container/collateral': {
'settings': {
'ANYTHING': {
'value': 'everything'
}
}
}
}
}
})
desc = query_dep(cfg)
assert (desc['application']['components']
['test-adjust-specific-container/collateral']['settings']
['ANYTHING']['value'] == 'everything')
cleanup_deployment(dep)
def test_adjust_env_var_setting():
dep = """
apiVersion: apps/v1
kind: Deployment
metadata:
name: test-adjust-env-var-setting
spec:
selector:
matchLabels:
app: test-adjust-env-var-setting
template:
metadata:
labels:
app: test-adjust-env-var-setting
spec:
containers:
- name: main
image: alpine:latest
command: ["/bin/sh", "-c", "sleep 3600"]
env:
- name: ANYTHING
value: everything
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
"""
cfg = """
k8s:
application:
components:
test-adjust-env-var-setting:
env:
ANYTHING:
values:
- everything
- nothing
"""
setup_deployment(dep)
adjust_dep(
cfg, {
'application': {
'components': {
'test-adjust-env-var-setting': {
'settings': {
'ANYTHING': {
'value': 'nothing'
}
}
}
}
}
})
desc = query_dep(cfg)
assert (desc['application']['components']['test-adjust-env-var-setting']
['settings']['ANYTHING']['value'] == 'nothing')
cleanup_deployment(dep)
def test_adjust_settings_mem_request_min_limit():
dep = """
apiVersion: apps/v1
kind: Deployment
metadata:
name: test-adjust-settings-mem
spec:
selector:
matchLabels:
app: test-adjust-settings-mem
template:
metadata:
labels:
app: test-adjust-settings-mem
spec:
containers:
- name: main
image: alpine:latest
command: ["/bin/sh", "-c", "sleep 3600"]
resources:
limits:
memory: .25Gi
"""
cfg = """
k8s:
application:
components:
test-adjust-settings-mem:
settings:
mem:
min: .125
max: .5
selector: request_min_limit
limit_min: .1875
"""
setup_deployment(dep)
adjust_dep(cfg, {'application': {'components': {'test-adjust-settings-mem': {'settings': {'mem': {'value': .125}}}}}})
desc = query_dep(cfg)
assert desc['application']['components']['test-adjust-settings-mem']['settings']['mem']['value'] == .125
# assert non-selector resource is cleared
all_deps = k_get('deployment')
dep_state = next(iter((i for i in all_deps['items'] if i['metadata']['name'] == 'test-adjust-settings-mem')))
assert dep_state['spec']['template']['spec']['containers'][0]['resources'].get('limits', {}).get('memory') == '192Mi'
adjust_dep(cfg, {'application': {'components': {'test-adjust-settings-mem': {'settings': {'mem': {'value': .25}}}}}})
desc = query_dep(cfg)
assert desc['application']['components']['test-adjust-settings-mem']['settings']['mem']['value'] == .25
all_deps = k_get('deployment')
dep_state = next(iter((i for i in all_deps['items'] if i['metadata']['name'] == 'test-adjust-settings-mem')))
assert dep_state['spec']['template']['spec']['containers'][0]['resources'].get('limits', {}).get('memory') == '256Mi'
cleanup_deployment(dep)
def test_encoder_adjust_command():
dep = """
apiVersion: apps/v1
kind: Deployment
metadata:
name: test-encoder-adjust-command
spec:
selector:
matchLabels:
app: test-encoder-adjust-command
template:
metadata:
labels:
app: test-encoder-adjust-command
spec:
containers:
- name: main
image: alpine:latest
command: ["/bin/sh", "-c", "sleep 3600", "&&", "echo", "-XX:GCTimeRatio=69"]
"""
cfg = """
k8s:
application:
components:
test-encoder-adjust-command:
command:
encoder:
name: jvm
before:
- /bin/sh
- -c
- sleep 3600
- "&&"
- echo
settings:
GCTimeRatio:
min: 9
max: 99
step: 10
"""
setup_deployment(dep)
adjust_dep(cfg, {'application': {
'components': {'test-encoder-adjust-command': {'settings': {'GCTimeRatio': {'value': 19}}}}}})
desc = query_dep(cfg)
assert desc['application']['components']['test-encoder-adjust-command']['settings']['GCTimeRatio']['value'] == 19
cleanup_deployment(dep)
def test_adjust_settings_cpu_limits():
dep = """
apiVersion: apps/v1
kind: Deployment
metadata:
name: test-adjust-settings-cpu
spec:
selector:
matchLabels:
app: test-adjust-settings-cpu
template:
metadata:
labels:
app: test-adjust-settings-cpu
spec:
containers:
- name: main
image: alpine:latest
command: ["/bin/sh", "-c", "sleep 3600"]
resources:
requests:
cpu: .3
"""
cfg = """
k8s:
application:
components:
test-adjust-settings-cpu:
settings:
cpu:
min: .125
max: .5
selector: limit
"""
setup_deployment(dep)
adjust_dep(cfg, {'application': {'components': {'test-adjust-settings-cpu': {'settings': {'cpu': {'value': .25}}}}}})
desc = query_dep(cfg)
assert desc['application']['components']['test-adjust-settings-cpu']['settings']['cpu']['value'] == .25
# assert non-selector resource is cleared
all_deps = k_get('deployment')
dep_state = next(iter((i for i in all_deps['items'] if i['metadata']['name'] == 'test-adjust-settings-cpu')))
assert dep_state['spec']['template']['spec']['containers'][0]['resources'].get('requests', {}).get('cpu') is None
adjust_dep(cfg, {'application': {'components': {'test-adjust-settings-cpu': {'settings': {'cpu': {'value': .125}}}}}})
desc = query_dep(cfg)
assert desc['application']['components']['test-adjust-settings-cpu']['settings']['cpu']['value'] == .125
cleanup_deployment(dep)
def test_encoder_adjust_env_var():
dep = """
apiVersion: apps/v1
kind: Deployment
metadata:
name: test-encoder-adjust-env-var
spec:
selector:
matchLabels:
app: test-encoder-adjust-env-var
template:
metadata:
labels:
app: test-encoder-adjust-env-var
spec:
containers:
- name: main
image: alpine:latest
command: ["/bin/sh", "-c", "sleep 3600"]
env:
- name: JAVA_OPTS
value: -XX:GCTimeRatio=69
"""
cfg = """
k8s:
application:
components:
test-encoder-adjust-env-var:
env:
JAVA_OPTS:
encoder:
name: jvm
settings:
GCTimeRatio:
min: 9
max: 99
step: 10
"""
setup_deployment(dep)
adjust_dep(cfg, {'application': {
'components': {'test-encoder-adjust-env-var': {'settings': {'GCTimeRatio': {'value': 19}}}}}})
desc = query_dep(cfg)
assert desc['application']['components']['test-encoder-adjust-env-var']['settings']['GCTimeRatio']['value'] == 19
cleanup_deployment(dep)
def test_adjust_settings_mem():
dep = """
apiVersion: apps/v1
kind: Deployment
metadata:
name: test-adjust-settings-mem
spec:
selector:
matchLabels:
app: test-adjust-settings-mem
template:
metadata:
labels:
app: test-adjust-settings-mem
spec:
containers:
- name: main
image: alpine:latest
command: ["/bin/sh", "-c", "sleep 3600"]
resources:
limits:
memory: .25Gi
"""
cfg = """
k8s:
application:
components:
test-adjust-settings-mem:
settings:
mem:
min: .1
max: .5
step: .1
"""
setup_deployment(dep)
adjust_dep(cfg,
{'application': {'components': {'test-adjust-settings-mem': {'settings': {'mem': {'value': .125}}}}}})
desc = query_dep(cfg)
assert desc['application']['components']['test-adjust-settings-mem']['settings']['mem']['value'] == .125
cleanup_deployment(dep)
def test_adjust_settings_deployment_config():
dep = """
apiVersion: apps/v1
kind: Deployment
metadata:
name: test-adjust-settings-cpu
spec:
selector:
matchLabels:
app: test-adjust-settings-cpu
template:
metadata:
labels:
app: test-adjust-settings-cpu
spec:
containers:
- name: main
image: alpine:latest
command: ["/bin/sh", "-c", "sleep 3600"]
resources:
limits:
cpu: .3
"""
cfg = """
k8s:
application:
components:
canary:
deployment: test-adjust-settings-cpu/main
settings:
cpu:
min: .1
max: .5
step: .1
"""
setup_deployment(dep)
adjust_dep(cfg, {'application': {'components': {'canary': {'settings': {'cpu': {'value': .2}}}}}})
desc = query_dep(cfg)
assert desc['application']['components']['canary']['settings']['cpu']['value'] == .2
cleanup_deployment(dep)
def test_adjust_settings_replicas():
dep = """
apiVersion: apps/v1
kind: Deployment
metadata:
name: test-adjust-settings-replicas
spec:
replicas: 2
selector:
matchLabels:
app: test-adjust-settings-replicas
template:
metadata:
labels:
app: test-adjust-settings-replicas
spec:
containers:
- name: main
image: alpine:latest
command: ["/bin/sh", "-c", "sleep 3600"]
"""
cfg = """
k8s:
application:
components:
test-adjust-settings-replicas:
settings:
replicas:
min: 1
max: 2
step: 1
"""
setup_deployment(dep)
adjust_dep(cfg,
{'application': {
'components': {'test-adjust-settings-replicas': {'settings': {'replicas': {'value': 1}}}}}})
desc = query_dep(cfg)
assert desc['application']['components']['test-adjust-settings-replicas']['settings']['replicas']['value'] == 1
cleanup_deployment(dep)
def test_query_settings_cpu():
dep = """
apiVersion: apps/v1
kind: Deployment
metadata:
name: test-query-settings-cpu
spec:
selector:
matchLabels:
app: test-query-settings-cpu
template:
metadata:
labels:
app: test-query-settings-cpu
spec:
containers:
- name: main
image: alpine:latest
command: ["/bin/sh", "-c", "sleep 3600"]
resources:
limits:
cpu: .3
"""
cfg = """
k8s:
application:
components:
test-query-settings-cpu:
settings:
cpu:
min: .1
max: .5
step: .1
"""
setup_deployment(dep)
desc = query_dep(cfg)
assert desc['application']['components']['test-query-settings-cpu']['settings']['cpu']['value'] == .3
cleanup_deployment(dep)
def test_encoder_adjust_with_setting_prefix():
dep = """
apiVersion: apps/v1
kind: Deployment
metadata:
name: test-encoder-adjust-with-setting-prefix
spec:
selector:
matchLabels:
app: test-encoder-adjust-with-setting-prefix
template:
metadata:
labels:
app: test-encoder-adjust-with-setting-prefix
spec:
containers:
- name: main
image: alpine:latest
command: ["/bin/sh", "-c", "sleep 3600", "&&", "echo", "-XX:GCTimeRatio=69"]
env:
- name: JAVA_OPTS
value: -XX:GCTimeRatio=69
"""
cfg = """
k8s:
application:
components:
test-encoder-adjust-with-setting-prefix:
command:
encoder:
name: jvm
before:
- /bin/sh
- -c
- sleep 3600
- "&&"
- echo
setting_prefix: cmd-
settings:
GCTimeRatio:
min: 9
max: 99
step: 10
env:
JAVA_OPTS:
encoder:
name: jvm
setting_prefix: env-
settings:
GCTimeRatio:
min: 9
max: 99
step: 10
"""
setup_deployment(dep)
adjust_dep(cfg, {'application': {
'components': {'test-encoder-adjust-with-setting-prefix': {
'settings': {'cmd-GCTimeRatio': {'value': 19}, 'env-GCTimeRatio': {'value': 19}}}}}})
desc = query_dep(cfg)
assert (desc['application']['components']['test-encoder-adjust-with-setting-prefix']['settings']['cmd-GCTimeRatio']
['value'] == 19)
assert (desc['application']['components']['test-encoder-adjust-with-setting-prefix']['settings']['env-GCTimeRatio']
['value'] == 19)
cleanup_deployment(dep)
def test_adjust_never_ready():
"""
The following deployment will never be ready once mem is adjusted to 0.125Gi
the test verifies adjust ok is not reported in this case
"""
dep = """
apiVersion: apps/v1
kind: Deployment
metadata:
name: test-adjust-never-ready
spec:
selector:
matchLabels:
app: test-adjust-never-ready
strategy:
rollingUpdate:
maxSurge: 25%
maxUnavailable: 25%
type: RollingUpdate
template:
metadata:
labels:
app: test-adjust-never-ready
spec:
containers:
- name: main
image: opsani/co-http
command:
- bash
- -c
- "if [ $(cat /sys/fs/cgroup/memory/memory.limit_in_bytes) -gt 191058816 ]; then /usr/local/bin/http; else sleep 1d; fi"
resources:
requests:
cpu: "0.2"
memory: "256Mi"
limits:
cpu: "0.2"
memory: "256Mi"
readinessProbe:
failureThreshold: 1000000
httpGet:
path: /
port: 8080
scheme: HTTP
initialDelaySeconds: 30
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 5
"""
cfg = """
k8s:
application:
components:
test-adjust-never-ready:
settings:
mem:
min: .125
max: .5
step: .125
"""
setup_deployment(dep)
try:
captured_error = None
try:
adjust_dep(
cfg, {
'control': {
'timeout': 90
},
'application': {
'components': {
'test-adjust-never-ready': {
'settings': {
'mem': {
'value': .125
}
}
}
}
}
})
except Exception as e:
captured_error = e
assert captured_error is not None, 'Adjustment succeeded despite latest revision pods never becoming ready'
assert 'timed out waiting for replicas to come up' in str(
captured_error
), 'Adjustment error occurred but did not match the expected error. Error that occured: {}'.format(
captured_error)
assert 'Rollback succeeded' in str(
captured_error
), 'The expected error occured but rollback failed. Error: {}'.format(
captured_error)
except:
raise
finally:
cleanup_deployment(dep)
def test_adjust_conflicting_rs_labels():
"""
The following deployments have conflicting labels which cuased a failure in older driver versions
"""
dep = """
apiVersion: apps/v1
kind: Deployment
metadata:
name: {name}
spec:
replicas: {replicas}
selector:
matchLabels: {{ app: nginx{addLabel} }}
template:
metadata:
labels: {{ app: nginx{addLabel} }}
spec:
containers:
- name: nginx
image: nginx:1.7.9
ports:
- containerPort: 80
resources:
requests:
cpu: "250m"
memory: "256Mi"
limits:
cpu: "250m"
memory: "256Mi"
"""
cfg = """
k8s:
application:
components:
{name}:
settings:
mem:
min: .125
max: .5
step: .125
"""
can_dep = dep.format(name='nginx-canary',
replicas='1',
addLabel=', role: canary')
main_dep = dep.format(name='nginx-main', replicas='2', addLabel='')
setup_deployment(can_dep)
setup_deployment(main_dep)
try:
adjust_dep(
cfg.format(name='nginx-canary'), {
'control': {
'timeout': 90
},
'application': {
'components': {
'nginx-canary': {
'settings': {
'mem': {
'value': .125
}
}
}
}
}
}) # can dep revision 2
adjust_dep(
cfg.format(name='nginx-canary'), {
'control': {
'timeout': 90
},
'application': {
'components': {
'nginx-canary': {
'settings': {
'mem': {
'value': .25
}
}
}
}
}
}) # can dep revision 3
adjust_dep(
cfg.format(name='nginx-main'), {
'control': {
'timeout': 90
},
'application': {
'components': {
'nginx-main': {
'settings': {
'mem': {
'value': .125
}
}
}
}
}
}) # actual test, main_dep revision 2
except:
raise
finally:
cleanup_deployment(can_dep)
cleanup_deployment(main_dep)
def test_adjust(simple_app):
"""test adjust command"""
dep = """
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-deployment
spec:
replicas: 3
selector:
matchLabels:
app: nginx
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx:1.7.9
ports:
- containerPort: 80
"""
setup_deployment(dep)
time.sleep(15)
setcfg("config_basic.yaml")
exit_code, data = adjust("--describe", "default")
assert exit_code == 0
assert data and isinstance(data, dict)
# check expected result (single component)
assert len(data["application"]["components"]) == 1
# save the initial 'monitoring' data
mon = data["monitoring"] # must be present
# run adjust (changes replicas to 2 from the initial 3)
exit_code, data = adjust("default", stdin=rep2)
mon2 = data["monitoring"]
# spec & version should not be affected by the adjust, nor ref app run ID
assert mon["spec_id"] == mon2["spec_id"]
assert mon["version_id"] == mon2["version_id"]
assert mon["ref_spec_id"] == mon2["ref_spec_id"]
assert mon["ref_version_id"] == mon2["ref_version_id"]
assert mon["ref_runtime_id"] == mon2["ref_runtime_id"]
# however, run id should change (different number of replicas)
assert mon["runtime_id"] != mon2["runtime_id"]
# check again with 'describe'
exit_code, data = adjust("--describe", "default")
mon2 = data["monitoring"]
# spec & version should not be affected by the adjust
assert mon["spec_id"] == mon2["spec_id"]
assert mon["version_id"] == mon2["version_id"]
assert mon["ref_spec_id"] == mon2["ref_spec_id"]
assert mon["ref_version_id"] == mon2["ref_version_id"]
cleanup_deployment(dep)
def test_adjust_destroy_new():
"""
The following deployment will never be ready once mem is adjusted to 0.125Gi
the test verifies adjust ok is not reported in this case
"""
dep = """
apiVersion: apps/v1
kind: Deployment
metadata:
name: test-adjust-destroy-new
spec:
selector:
matchLabels:
app: test-adjust-destroy-new
strategy:
rollingUpdate:
maxSurge: 100%
maxUnavailable: 0%
type: RollingUpdate
template:
metadata:
labels:
app: test-adjust-destroy-new
spec:
containers:
- name: main
image: opsani/co-http
command:
- bash
- -c
- "if [ $(cat /sys/fs/cgroup/memory/memory.limit_in_bytes) -gt 191058816 ]; then /usr/local/bin/http; else sleep 1d; fi"
resources:
requests:
cpu: "0.2"
memory: "256Mi"
limits:
cpu: "0.2"
memory: "256Mi"
readinessProbe:
failureThreshold: 3
httpGet:
path: /
port: 8080
scheme: HTTP
initialDelaySeconds: 30
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 5
livenessProbe:
failureThreshold: 3
httpGet:
path: /
port: 8080
scheme: HTTP
initialDelaySeconds: 30
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 5
"""
cfg = """
k8s:
on_fail: destroy_new
application:
components:
test-adjust-destroy-new:
settings:
mem:
min: .125
max: .5
step: .125
"""
setup_deployment(dep)
time.sleep(40) # let initial dep become ready before testing url
run('kubectl expose deployment test-adjust-destroy-new --type=LoadBalancer --port=8080'
)
url = run('minikube service test-adjust-destroy-new --url')
time.sleep(15) # give time for service to come up
try:
connection_error = None
def test_url():
nonlocal connection_error
try:
resp = requests.get(url)
except requests.ConnectionError as e:
connection_error = 'Test deployment became unreachable: {}'.format(
e)
else:
if not resp.ok:
connection_error = 'Test deployment became unreachable, status {}: {}'.format(
resp.status_code, resp.text)
timer = Timer(1, test_url)
timer.daemon = True
timer.start()
timer = Timer(1, test_url)
timer.daemon = True
timer.start()
time.sleep(5) # verify connection of initial pod
assert connection_error is None, f"Pre-adjust: {connection_error}"
captured_error = None
try:
adjust_dep(
cfg, {
'application': {
'components': {
'test-adjust-destroy-new': {
'settings': {
'mem': {
'value': .125
}
}
}
}
}
})
except Exception as e:
captured_error = e
assert captured_error is not None, 'Adjustment succeeded despite latest revision pods never becoming ready'
assert 'during rollout; component(s) crash restart detected' in str(
captured_error
), 'Adjustment error occurred but did not match the expected error. Error that occured: {}'.format(
captured_error)
assert 'Rollback succeeded' in str(
captured_error
), 'The expected error occured but rollback failed. Error: {}'.format(
captured_error)
assert connection_error is None, connection_error
except:
raise
finally:
cleanup_deployment(dep)
run('kubectl delete service test-adjust-destroy-new')
