def test_adjust_specific_container(): dep = """ apiVersion: apps/v1 kind: Deployment metadata: name: test-adjust-specific-container spec: selector: matchLabels: app: test-adjust-specific-container template: metadata: labels: app: test-adjust-specific-container spec: containers: - name: main image: alpine:latest command: ["/bin/sh", "-c", "sleep 3600"] env: - name: ANYTHING value: everything - name: collateral image: alpine:latest command: ["/bin/sh", "-c", "sleep 3600"] env: - name: ANYTHING value: nothing """ cfg = """ k8s: application: components: test-adjust-specific-container/collateral: env: ANYTHING: values: - everything - nothing """ setup_deployment(dep) adjust_dep( cfg, { 'application': { 'components': { 'test-adjust-specific-container/collateral': { 'settings': { 'ANYTHING': { 'value': 'everything' } } } } } }) desc = query_dep(cfg) assert (desc['application']['components'] ['test-adjust-specific-container/collateral']['settings'] ['ANYTHING']['value'] == 'everything') cleanup_deployment(dep)
def test_adjust_env_var_setting(): dep = """ apiVersion: apps/v1 kind: Deployment metadata: name: test-adjust-env-var-setting spec: selector: matchLabels: app: test-adjust-env-var-setting template: metadata: labels: app: test-adjust-env-var-setting spec: containers: - name: main image: alpine:latest command: ["/bin/sh", "-c", "sleep 3600"] env: - name: ANYTHING value: everything - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName """ cfg = """ k8s: application: components: test-adjust-env-var-setting: env: ANYTHING: values: - everything - nothing """ setup_deployment(dep) adjust_dep( cfg, { 'application': { 'components': { 'test-adjust-env-var-setting': { 'settings': { 'ANYTHING': { 'value': 'nothing' } } } } } }) desc = query_dep(cfg) assert (desc['application']['components']['test-adjust-env-var-setting'] ['settings']['ANYTHING']['value'] == 'nothing') cleanup_deployment(dep)
def test_adjust_settings_mem_request_min_limit(): dep = """ apiVersion: apps/v1 kind: Deployment metadata: name: test-adjust-settings-mem spec: selector: matchLabels: app: test-adjust-settings-mem template: metadata: labels: app: test-adjust-settings-mem spec: containers: - name: main image: alpine:latest command: ["/bin/sh", "-c", "sleep 3600"] resources: limits: memory: .25Gi """ cfg = """ k8s: application: components: test-adjust-settings-mem: settings: mem: min: .125 max: .5 selector: request_min_limit limit_min: .1875 """ setup_deployment(dep) adjust_dep(cfg, {'application': {'components': {'test-adjust-settings-mem': {'settings': {'mem': {'value': .125}}}}}}) desc = query_dep(cfg) assert desc['application']['components']['test-adjust-settings-mem']['settings']['mem']['value'] == .125 # assert non-selector resource is cleared all_deps = k_get('deployment') dep_state = next(iter((i for i in all_deps['items'] if i['metadata']['name'] == 'test-adjust-settings-mem'))) assert dep_state['spec']['template']['spec']['containers'][0]['resources'].get('limits', {}).get('memory') == '192Mi' adjust_dep(cfg, {'application': {'components': {'test-adjust-settings-mem': {'settings': {'mem': {'value': .25}}}}}}) desc = query_dep(cfg) assert desc['application']['components']['test-adjust-settings-mem']['settings']['mem']['value'] == .25 all_deps = k_get('deployment') dep_state = next(iter((i for i in all_deps['items'] if i['metadata']['name'] == 'test-adjust-settings-mem'))) assert dep_state['spec']['template']['spec']['containers'][0]['resources'].get('limits', {}).get('memory') == '256Mi' cleanup_deployment(dep)
def test_encoder_adjust_command(): dep = """ apiVersion: apps/v1 kind: Deployment metadata: name: test-encoder-adjust-command spec: selector: matchLabels: app: test-encoder-adjust-command template: metadata: labels: app: test-encoder-adjust-command spec: containers: - name: main image: alpine:latest command: ["/bin/sh", "-c", "sleep 3600", "&&", "echo", "-XX:GCTimeRatio=69"] """ cfg = """ k8s: application: components: test-encoder-adjust-command: command: encoder: name: jvm before: - /bin/sh - -c - sleep 3600 - "&&" - echo settings: GCTimeRatio: min: 9 max: 99 step: 10 """ setup_deployment(dep) adjust_dep(cfg, {'application': { 'components': {'test-encoder-adjust-command': {'settings': {'GCTimeRatio': {'value': 19}}}}}}) desc = query_dep(cfg) assert desc['application']['components']['test-encoder-adjust-command']['settings']['GCTimeRatio']['value'] == 19 cleanup_deployment(dep)
def test_adjust_settings_cpu_limits(): dep = """ apiVersion: apps/v1 kind: Deployment metadata: name: test-adjust-settings-cpu spec: selector: matchLabels: app: test-adjust-settings-cpu template: metadata: labels: app: test-adjust-settings-cpu spec: containers: - name: main image: alpine:latest command: ["/bin/sh", "-c", "sleep 3600"] resources: requests: cpu: .3 """ cfg = """ k8s: application: components: test-adjust-settings-cpu: settings: cpu: min: .125 max: .5 selector: limit """ setup_deployment(dep) adjust_dep(cfg, {'application': {'components': {'test-adjust-settings-cpu': {'settings': {'cpu': {'value': .25}}}}}}) desc = query_dep(cfg) assert desc['application']['components']['test-adjust-settings-cpu']['settings']['cpu']['value'] == .25 # assert non-selector resource is cleared all_deps = k_get('deployment') dep_state = next(iter((i for i in all_deps['items'] if i['metadata']['name'] == 'test-adjust-settings-cpu'))) assert dep_state['spec']['template']['spec']['containers'][0]['resources'].get('requests', {}).get('cpu') is None adjust_dep(cfg, {'application': {'components': {'test-adjust-settings-cpu': {'settings': {'cpu': {'value': .125}}}}}}) desc = query_dep(cfg) assert desc['application']['components']['test-adjust-settings-cpu']['settings']['cpu']['value'] == .125 cleanup_deployment(dep)
def test_encoder_adjust_env_var(): dep = """ apiVersion: apps/v1 kind: Deployment metadata: name: test-encoder-adjust-env-var spec: selector: matchLabels: app: test-encoder-adjust-env-var template: metadata: labels: app: test-encoder-adjust-env-var spec: containers: - name: main image: alpine:latest command: ["/bin/sh", "-c", "sleep 3600"] env: - name: JAVA_OPTS value: -XX:GCTimeRatio=69 """ cfg = """ k8s: application: components: test-encoder-adjust-env-var: env: JAVA_OPTS: encoder: name: jvm settings: GCTimeRatio: min: 9 max: 99 step: 10 """ setup_deployment(dep) adjust_dep(cfg, {'application': { 'components': {'test-encoder-adjust-env-var': {'settings': {'GCTimeRatio': {'value': 19}}}}}}) desc = query_dep(cfg) assert desc['application']['components']['test-encoder-adjust-env-var']['settings']['GCTimeRatio']['value'] == 19 cleanup_deployment(dep)
def test_adjust_settings_mem(): dep = """ apiVersion: apps/v1 kind: Deployment metadata: name: test-adjust-settings-mem spec: selector: matchLabels: app: test-adjust-settings-mem template: metadata: labels: app: test-adjust-settings-mem spec: containers: - name: main image: alpine:latest command: ["/bin/sh", "-c", "sleep 3600"] resources: limits: memory: .25Gi """ cfg = """ k8s: application: components: test-adjust-settings-mem: settings: mem: min: .1 max: .5 step: .1 """ setup_deployment(dep) adjust_dep(cfg, {'application': {'components': {'test-adjust-settings-mem': {'settings': {'mem': {'value': .125}}}}}}) desc = query_dep(cfg) assert desc['application']['components']['test-adjust-settings-mem']['settings']['mem']['value'] == .125 cleanup_deployment(dep)
def test_adjust_settings_deployment_config(): dep = """ apiVersion: apps/v1 kind: Deployment metadata: name: test-adjust-settings-cpu spec: selector: matchLabels: app: test-adjust-settings-cpu template: metadata: labels: app: test-adjust-settings-cpu spec: containers: - name: main image: alpine:latest command: ["/bin/sh", "-c", "sleep 3600"] resources: limits: cpu: .3 """ cfg = """ k8s: application: components: canary: deployment: test-adjust-settings-cpu/main settings: cpu: min: .1 max: .5 step: .1 """ setup_deployment(dep) adjust_dep(cfg, {'application': {'components': {'canary': {'settings': {'cpu': {'value': .2}}}}}}) desc = query_dep(cfg) assert desc['application']['components']['canary']['settings']['cpu']['value'] == .2 cleanup_deployment(dep)
def test_adjust_settings_replicas(): dep = """ apiVersion: apps/v1 kind: Deployment metadata: name: test-adjust-settings-replicas spec: replicas: 2 selector: matchLabels: app: test-adjust-settings-replicas template: metadata: labels: app: test-adjust-settings-replicas spec: containers: - name: main image: alpine:latest command: ["/bin/sh", "-c", "sleep 3600"] """ cfg = """ k8s: application: components: test-adjust-settings-replicas: settings: replicas: min: 1 max: 2 step: 1 """ setup_deployment(dep) adjust_dep(cfg, {'application': { 'components': {'test-adjust-settings-replicas': {'settings': {'replicas': {'value': 1}}}}}}) desc = query_dep(cfg) assert desc['application']['components']['test-adjust-settings-replicas']['settings']['replicas']['value'] == 1 cleanup_deployment(dep)
def test_query_settings_cpu(): dep = """ apiVersion: apps/v1 kind: Deployment metadata: name: test-query-settings-cpu spec: selector: matchLabels: app: test-query-settings-cpu template: metadata: labels: app: test-query-settings-cpu spec: containers: - name: main image: alpine:latest command: ["/bin/sh", "-c", "sleep 3600"] resources: limits: cpu: .3 """ cfg = """ k8s: application: components: test-query-settings-cpu: settings: cpu: min: .1 max: .5 step: .1 """ setup_deployment(dep) desc = query_dep(cfg) assert desc['application']['components']['test-query-settings-cpu']['settings']['cpu']['value'] == .3 cleanup_deployment(dep)
def test_encoder_adjust_with_setting_prefix(): dep = """ apiVersion: apps/v1 kind: Deployment metadata: name: test-encoder-adjust-with-setting-prefix spec: selector: matchLabels: app: test-encoder-adjust-with-setting-prefix template: metadata: labels: app: test-encoder-adjust-with-setting-prefix spec: containers: - name: main image: alpine:latest command: ["/bin/sh", "-c", "sleep 3600", "&&", "echo", "-XX:GCTimeRatio=69"] env: - name: JAVA_OPTS value: -XX:GCTimeRatio=69 """ cfg = """ k8s: application: components: test-encoder-adjust-with-setting-prefix: command: encoder: name: jvm before: - /bin/sh - -c - sleep 3600 - "&&" - echo setting_prefix: cmd- settings: GCTimeRatio: min: 9 max: 99 step: 10 env: JAVA_OPTS: encoder: name: jvm setting_prefix: env- settings: GCTimeRatio: min: 9 max: 99 step: 10 """ setup_deployment(dep) adjust_dep(cfg, {'application': { 'components': {'test-encoder-adjust-with-setting-prefix': { 'settings': {'cmd-GCTimeRatio': {'value': 19}, 'env-GCTimeRatio': {'value': 19}}}}}}) desc = query_dep(cfg) assert (desc['application']['components']['test-encoder-adjust-with-setting-prefix']['settings']['cmd-GCTimeRatio'] ['value'] == 19) assert (desc['application']['components']['test-encoder-adjust-with-setting-prefix']['settings']['env-GCTimeRatio'] ['value'] == 19) cleanup_deployment(dep)
def test_adjust_never_ready(): """ The following deployment will never be ready once mem is adjusted to 0.125Gi the test verifies adjust ok is not reported in this case """ dep = """ apiVersion: apps/v1 kind: Deployment metadata: name: test-adjust-never-ready spec: selector: matchLabels: app: test-adjust-never-ready strategy: rollingUpdate: maxSurge: 25% maxUnavailable: 25% type: RollingUpdate template: metadata: labels: app: test-adjust-never-ready spec: containers: - name: main image: opsani/co-http command: - bash - -c - "if [ $(cat /sys/fs/cgroup/memory/memory.limit_in_bytes) -gt 191058816 ]; then /usr/local/bin/http; else sleep 1d; fi" resources: requests: cpu: "0.2" memory: "256Mi" limits: cpu: "0.2" memory: "256Mi" readinessProbe: failureThreshold: 1000000 httpGet: path: / port: 8080 scheme: HTTP initialDelaySeconds: 30 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 5 """ cfg = """ k8s: application: components: test-adjust-never-ready: settings: mem: min: .125 max: .5 step: .125 """ setup_deployment(dep) try: captured_error = None try: adjust_dep( cfg, { 'control': { 'timeout': 90 }, 'application': { 'components': { 'test-adjust-never-ready': { 'settings': { 'mem': { 'value': .125 } } } } } }) except Exception as e: captured_error = e assert captured_error is not None, 'Adjustment succeeded despite latest revision pods never becoming ready' assert 'timed out waiting for replicas to come up' in str( captured_error ), 'Adjustment error occurred but did not match the expected error. Error that occured: {}'.format( captured_error) assert 'Rollback succeeded' in str( captured_error ), 'The expected error occured but rollback failed. Error: {}'.format( captured_error) except: raise finally: cleanup_deployment(dep)
def test_adjust_conflicting_rs_labels(): """ The following deployments have conflicting labels which cuased a failure in older driver versions """ dep = """ apiVersion: apps/v1 kind: Deployment metadata: name: {name} spec: replicas: {replicas} selector: matchLabels: {{ app: nginx{addLabel} }} template: metadata: labels: {{ app: nginx{addLabel} }} spec: containers: - name: nginx image: nginx:1.7.9 ports: - containerPort: 80 resources: requests: cpu: "250m" memory: "256Mi" limits: cpu: "250m" memory: "256Mi" """ cfg = """ k8s: application: components: {name}: settings: mem: min: .125 max: .5 step: .125 """ can_dep = dep.format(name='nginx-canary', replicas='1', addLabel=', role: canary') main_dep = dep.format(name='nginx-main', replicas='2', addLabel='') setup_deployment(can_dep) setup_deployment(main_dep) try: adjust_dep( cfg.format(name='nginx-canary'), { 'control': { 'timeout': 90 }, 'application': { 'components': { 'nginx-canary': { 'settings': { 'mem': { 'value': .125 } } } } } }) # can dep revision 2 adjust_dep( cfg.format(name='nginx-canary'), { 'control': { 'timeout': 90 }, 'application': { 'components': { 'nginx-canary': { 'settings': { 'mem': { 'value': .25 } } } } } }) # can dep revision 3 adjust_dep( cfg.format(name='nginx-main'), { 'control': { 'timeout': 90 }, 'application': { 'components': { 'nginx-main': { 'settings': { 'mem': { 'value': .125 } } } } } }) # actual test, main_dep revision 2 except: raise finally: cleanup_deployment(can_dep) cleanup_deployment(main_dep)
def test_adjust(simple_app): """test adjust command""" dep = """ apiVersion: apps/v1 kind: Deployment metadata: name: nginx-deployment spec: replicas: 3 selector: matchLabels: app: nginx template: metadata: labels: app: nginx spec: containers: - name: nginx image: nginx:1.7.9 ports: - containerPort: 80 """ setup_deployment(dep) time.sleep(15) setcfg("config_basic.yaml") exit_code, data = adjust("--describe", "default") assert exit_code == 0 assert data and isinstance(data, dict) # check expected result (single component) assert len(data["application"]["components"]) == 1 # save the initial 'monitoring' data mon = data["monitoring"] # must be present # run adjust (changes replicas to 2 from the initial 3) exit_code, data = adjust("default", stdin=rep2) mon2 = data["monitoring"] # spec & version should not be affected by the adjust, nor ref app run ID assert mon["spec_id"] == mon2["spec_id"] assert mon["version_id"] == mon2["version_id"] assert mon["ref_spec_id"] == mon2["ref_spec_id"] assert mon["ref_version_id"] == mon2["ref_version_id"] assert mon["ref_runtime_id"] == mon2["ref_runtime_id"] # however, run id should change (different number of replicas) assert mon["runtime_id"] != mon2["runtime_id"] # check again with 'describe' exit_code, data = adjust("--describe", "default") mon2 = data["monitoring"] # spec & version should not be affected by the adjust assert mon["spec_id"] == mon2["spec_id"] assert mon["version_id"] == mon2["version_id"] assert mon["ref_spec_id"] == mon2["ref_spec_id"] assert mon["ref_version_id"] == mon2["ref_version_id"] cleanup_deployment(dep)
def test_adjust_destroy_new(): """ The following deployment will never be ready once mem is adjusted to 0.125Gi the test verifies adjust ok is not reported in this case """ dep = """ apiVersion: apps/v1 kind: Deployment metadata: name: test-adjust-destroy-new spec: selector: matchLabels: app: test-adjust-destroy-new strategy: rollingUpdate: maxSurge: 100% maxUnavailable: 0% type: RollingUpdate template: metadata: labels: app: test-adjust-destroy-new spec: containers: - name: main image: opsani/co-http command: - bash - -c - "if [ $(cat /sys/fs/cgroup/memory/memory.limit_in_bytes) -gt 191058816 ]; then /usr/local/bin/http; else sleep 1d; fi" resources: requests: cpu: "0.2" memory: "256Mi" limits: cpu: "0.2" memory: "256Mi" readinessProbe: failureThreshold: 3 httpGet: path: / port: 8080 scheme: HTTP initialDelaySeconds: 30 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 5 livenessProbe: failureThreshold: 3 httpGet: path: / port: 8080 scheme: HTTP initialDelaySeconds: 30 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 5 """ cfg = """ k8s: on_fail: destroy_new application: components: test-adjust-destroy-new: settings: mem: min: .125 max: .5 step: .125 """ setup_deployment(dep) time.sleep(40) # let initial dep become ready before testing url run('kubectl expose deployment test-adjust-destroy-new --type=LoadBalancer --port=8080' ) url = run('minikube service test-adjust-destroy-new --url') time.sleep(15) # give time for service to come up try: connection_error = None def test_url(): nonlocal connection_error try: resp = requests.get(url) except requests.ConnectionError as e: connection_error = 'Test deployment became unreachable: {}'.format( e) else: if not resp.ok: connection_error = 'Test deployment became unreachable, status {}: {}'.format( resp.status_code, resp.text) timer = Timer(1, test_url) timer.daemon = True timer.start() timer = Timer(1, test_url) timer.daemon = True timer.start() time.sleep(5) # verify connection of initial pod assert connection_error is None, f"Pre-adjust: {connection_error}" captured_error = None try: adjust_dep( cfg, { 'application': { 'components': { 'test-adjust-destroy-new': { 'settings': { 'mem': { 'value': .125 } } } } } }) except Exception as e: captured_error = e assert captured_error is not None, 'Adjustment succeeded despite latest revision pods never becoming ready' assert 'during rollout; component(s) crash restart detected' in str( captured_error ), 'Adjustment error occurred but did not match the expected error. Error that occured: {}'.format( captured_error) assert 'Rollback succeeded' in str( captured_error ), 'The expected error occured but rollback failed. Error: {}'.format( captured_error) assert connection_error is None, connection_error except: raise finally: cleanup_deployment(dep) run('kubectl delete service test-adjust-destroy-new')