def test_tell_release_name():
_, content = run_under_click_context(tell_release_name, )
assert content == DUMMY_APPNAME
override_values = {'releaseName': DUMMY_OVERRIDE_RELEASE_NAME}
yadu(override_values, f'{CHART_DIR_NAME}/values-{TEST_CLUSTER}.yaml')
_, content = run_under_click_context(tell_release_name, )
assert content == DUMMY_OVERRIDE_RELEASE_NAME
def dummy(request):
def tear_down():
# 拆除测试的结果就不要要求这么高了, 因为有时候会打断点手动调试
# 跑这段拆除代码的时候, 可能东西已经被拆干净了
run(lain, args=['delete', '--purge'], returncode=None)
helm('delete', DUMMY_OVERRIDE_RELEASE_NAME, check=False)
ensure_absent([
CHART_DIR_NAME,
join(TESTS_BASE_DIR, DUMMY_REPO, DOCKERFILE_NAME)
])
if not getcwd().endswith(DUMMY_REPO):
sys.path.append(TESTS_BASE_DIR)
chdir(DUMMY_REPO)
tear_down()
run(lain, args=['init'])
override_values_for_e2e = {
'deployments': {
'web': {
'terminationGracePeriodSeconds': 1
}
}
}
override_values_file = f'values-{TEST_CLUSTER}.yaml'
yadu(override_values_for_e2e, join(CHART_DIR_NAME, override_values_file))
# `lain secret show` will create a dummy secret
run(lain, args=['secret', 'show'])
request.addfinalizer(tear_down)
def test_schemas():
bare_values = load_dummy_values()
web_proc = bare_values['deployments']['web']
# deploy is an alias for deployments
bare_values['deploy'] = {'web': web_proc, 'another': web_proc}
yadu(bare_values, DUMMY_VALUES_PATH)
_, values = run_under_click_context(load_helm_values,
(DUMMY_VALUES_PATH, ))
assert values['deployments']['web'] == values['deployments']['another']
assert values['cronjobs'] == {}
build = values['build']
assert build['prepare']['keep'] == [f'./{BUILD_TREASURE_NAME}']
bare_values['volumeMounts'][0]['subPath'] = 'foo/bar' # should be basename
with pytest.raises(ValidationError) as e:
HelmValuesSchema().load(bare_values)
assert 'subPath should be' in str(e)
false_ing = {'host': 'dummy', 'deployName': 'web'}
with pytest.raises(ValidationError):
IngressSchema().load(false_ing)
bad_web = {'containerPort': 8000}
with pytest.raises(ValidationError):
IngressSchema().load(bad_web)
def test_cluster_values_override():
fake_registry = 'registry.example.com'
override_values = {
'registry': fake_registry,
}
yadu(override_values, f'{CHART_DIR_NAME}/values-{TEST_CLUSTER}.yaml')
_, cc = run_under_click_context(tell_cluster_config, )
assert cc['registry'] == fake_registry
def test_tell_job_names():
override_values = {
'jobs': DUMMY_JOBS_CLAUSE,
'tests': DUMMY_TESTS_CLAUSE,
}
yadu(override_values, f'{CHART_DIR_NAME}/values-{TEST_CLUSTER}.yaml')
_, content = run_under_click_context(tell_job_names, )
assert set(content) == {'dummy-init'}
def test_ya():
dic = {'slogan': BULLSHIT}
f = NamedTemporaryFile()
yadu(dic, f)
f.seek(0)
assert yalo(f) == dic
multiline_content = {'so': LiteralScalarString('so\nlong')}
s = yadu(multiline_content)
# should dump multiline string in readable format
assert ': |' in s
def with_extra_values_file():
obj = context().obj
dic = {'labels': {'foo': 'bar'}}
f = NamedTemporaryFile(prefix='values-extra', suffix='.yaml')
yadu(dic, f)
f.seek(0)
obj['extra_values_file'] = f
try:
return tell_helm_options()
finally:
del f
def test_canary():
res = run(lain, args=['deploy', '--canary'], returncode=1)
assert 'cannot initiate canary deploy' in ensure_str(res.output)
run(lain, args=['deploy'])
res = run(lain, args=['deploy', '--canary'])
assert 'canary version has been deployed' in ensure_str(res.output)
res = run(lain, args=['deploy'], returncode=1)
assert 'cannot proceed due to on-going canary deploy' in ensure_str(
res.output)
resp = url_get_json(DUMMY_URL)
assert resp['env']['HOSTNAME'].startswith(f'{DUMMY_APPNAME}-web')
res = run(lain, args=['set-canary-group', 'internal'], returncode=1)
assert 'canaryGroups not defined in values' in ensure_str(res.output)
# inject canary annotations for test purpose
values = load_dummy_values()
canary_header_name = 'canary'
values['canaryGroups'] = {
'internal': {
'nginx.ingress.kubernetes.io/canary-by-header': canary_header_name
},
}
yadu(values, DUMMY_VALUES_PATH)
run(lain, args=['set-canary-group', 'internal'])
ings_res = kubectl(
'get',
'ing',
'-ojson',
'-l',
f'helm.sh/chart={DUMMY_CANARY_NAME}',
capture_output=True,
)
ings = jalo(ings_res.stdout)
for ing in ings['items']:
annotations = ing['metadata']['annotations']
assert (annotations['nginx.ingress.kubernetes.io/canary-by-header'] ==
canary_header_name)
canary_header = {canary_header_name: 'always'}
resp = url_get_json(DUMMY_URL, headers=canary_header)
assert resp['env']['HOSTNAME'].startswith(f'{DUMMY_CANARY_NAME}-web')
run(lain, args=['set-canary-group', '--abort'])
run(lain, args=['wait'])
assert f'{DUMMY_CANARY_NAME}-web' not in get_dummy_pod_names()
values['tests'] = DUMMY_TESTS_CLAUSE
yadu(values, DUMMY_VALUES_PATH)
tag = 'latest'
run(lain, args=['deploy', '--set', f'imageTag={tag}', '--canary'])
run(lain, args=['set-canary-group', '--final'])
run(lain, args=['wait'])
assert f'{DUMMY_CANARY_NAME}-web' not in get_dummy_pod_names()
image = get_deploy_image(f'{DUMMY_APPNAME}-web')
assert image.endswith(f':{tag}')
def test_sts():
# sts values are mostly the same with deploy, we just have to change a few
# things to make it work
values = load_dummy_values()
sts = deepcopy(values['deployments']['web'])
override_values = {
'statefulSets': {
'worker': sts,
}
}
yadu(override_values, f'{CHART_DIR_NAME}/values-{TEST_CLUSTER}.yaml')
run(lain, args=['deploy'])
_, pod_name = run_under_click_context(pick_pod,
kwargs={'proc_name': 'worker'})
assert pod_name == f'{DUMMY_APPNAME}-worker-0'
def test_load_helm_values():
# test internal cluster values are correctly loaded
_, values = run_under_click_context(load_helm_values, )
assert values['registry'] == 'docker.io/timfeirg'
assert values['domain'] == 'info'
dummy_jobs = {
'init': {
'command': ['echo', 'nothing']
},
}
override_values = {
'jobs': dummy_jobs,
}
yadu(override_values, f'{CHART_DIR_NAME}/values-{TEST_CLUSTER}.yaml')
_, values = run_under_click_context(load_helm_values, )
assert values['jobs'] == dummy_jobs
def test_override_release_name():
override_values_file_path = join(CHART_DIR_NAME, 'values-override.yaml')
override_values = {
'releaseName':
DUMMY_OVERRIDE_RELEASE_NAME,
'ingresses': [{
'host': DUMMY_OVERRIDE_RELEASE_NAME,
'deployName': 'web',
'paths': ['/'],
}],
}
yadu(override_values, override_values_file_path)
override_args = ['-f', override_values_file_path]
run(lain, args=override_args + ['deploy'])
status_dic = helm_status(DUMMY_OVERRIDE_RELEASE_NAME)
# helm release name should be correctly overridden
assert status_dic['name'] == DUMMY_OVERRIDE_RELEASE_NAME
# deploy a 'normal' version, to assure two releases do not interfere
run(lain, args=['deploy', '--wait'])
# get pods by appname, rather than releaseName
_, pods = get_pods(appname=DUMMY_APPNAME)
deploys = set()
for pod in pods:
pod_name, *_ = pod.split(None, 1)
if pod_name.endswith('test'):
# ignore test container
continue
deploy_name = tell_pod_deploy_name(pod_name)
if deploy_name == 'dummy':
# this is job pod, not deploy
continue
deploys.add(deploy_name)
assert deploys == {
f'{DUMMY_APPNAME}-web', f'{DUMMY_OVERRIDE_RELEASE_NAME}-web'
}
assert f'{DUMMY_OVERRIDE_RELEASE_NAME}-web' in ''.join(pods)
assert f'{DUMMY_APPNAME}-web' in ''.join(pods)
res = run(lain, args=override_args + ['deploy', '--canary'], returncode=1)
assert 'do not use canary deploy while values are being overridden' in res.stdout
run(lain, args=override_args + ['delete'])
# overridden release is deleted, but the 'normal' app remains intact
assert not helm_status(DUMMY_OVERRIDE_RELEASE_NAME)
assert helm_status(DUMMY_APPNAME)['name'] == DUMMY_APPNAME
def test_tell_all_clusters(mocker):
test_cluster_values_file = f'values-{TEST_CLUSTER}.yaml'
# we need at least two clusters to verify that tell_all_clusters are working correctly
another_cluster_name = 'another'
another_cluster_values_file = f'values-{another_cluster_name}.yaml'
tempd = TemporaryDirectory()
test_cluster_values_path = join(CLUSTER_VALUES_DIR,
test_cluster_values_file)
test_cluster_values = yalo(test_cluster_values_path)
test_cluster_values['registry'] = 'another.example.com'
shutil.copyfile(test_cluster_values_path,
join(tempd.name, test_cluster_values_file))
yadu(test_cluster_values, join(tempd.name, another_cluster_values_file))
mocker.patch('lain_cli.utils.KUBECONFIG_DIR', tempd.name)
mocker.patch('lain_cli.utils.CLUSTER_VALUES_DIR', tempd.name)
# touch kubeconfig-another
Path(join(tempd.name, f'kubeconfig-{another_cluster_name}')).write_text('')
Path(join(tempd.name, f'kubeconfig-{TEST_CLUSTER}')).write_text('')
# now that kubeconfig and cluster values file are present, we can verify
# CLUSTERS is correct
_, ccs = run_under_click_context(tell_all_clusters, )
assert set(ccs) == {TEST_CLUSTER, another_cluster_name}
assert ccs['another']['registry'] == test_cluster_values['registry']
tempd.cleanup()
def test_workflow(registry):
# lain init should failed when chart directory already exists
run(lain, args=['init'], returncode=1)
# use -f to remove chart directory and redo
run(lain, args=['init', '-f'])
# lain use will switch current context switch to [TEST_CLUSTER]
run(lain, args=['use', TEST_CLUSTER])
# lain use will print current cluster
res = run(lain, args=['use'])
assert f'* {TEST_CLUSTER}' in ensure_str(res.stdout)
# this makes sure lain-use can work when kubeconfig is absent
ensure_absent(join(KUBECONFIG_DIR, 'config'))
run(lain, args=['use', TEST_CLUSTER])
# see if this image is actually present on registry
res = run(lain, args=['image'])
image_tag = res.stdout.strip().split(':')[-1]
# should fail when using a bad image tag
res = run(lain, args=['deploy', '--set', 'imageTag=noway'], returncode=1)
assert 'image not found' in ensure_str(res.output).lower()
cronjob_name = 'nothing'
override_values = {
# 随便加一个 job, 为了看下一次部署的时候能否顺利先清理掉这个 job
'jobs': DUMMY_JOBS_CLAUSE,
# 随便加一个 cronjob, 为了测试 lain create-job
'cronjobs': {
cronjob_name: {
'schedule': '0 0 * * *',
'command': ['echo', RANDOM_STRING],
},
},
}
yadu(override_values, f'{CHART_DIR_NAME}/values-{TEST_CLUSTER}.yaml')
# use a built image to deploy
run(lain,
args=['--ignore-lint', 'deploy', '--set', f'imageTag={image_tag}'])
res = run(lain, args=['create-job', cronjob_name])
create_job_cmd = f'kubectl create job --from=cronjob/{DUMMY_APPNAME}-{cronjob_name} manual-test-{cronjob_name}'
assert create_job_cmd in res.output
# check service is up
dummy_resp = url_get_json(DUMMY_URL)
assert dummy_resp['env']['FOO'] == 'BAR'
assert dummy_resp['secretfile'] == 'I\nAM\nBATMAN'
# check if hostAliases is working
assert 'localhost' in dummy_resp['hosts']
assert 'local' in dummy_resp['hosts']
# check imageTag is correct
deployed_images = tell_deployed_images(DUMMY_APPNAME)
assert len(deployed_images) == 1
deployed_image = deployed_images.pop()
assert deployed_image.endswith(image_tag)
# check if init job succeeded
wait_for_job_success()
# run a extra job, to test lain job functionalities
command = 'env'
res = run(lain, args=['job', '--force', command])
_, job_name = run_under_click_context(make_job_name, args=(command, ))
pod_name = wait_for_job_success(job_name)
_, pod_name_again = run_under_click_context(
pick_pod, kwargs={'selector': f'job-name={job_name}'})
# check if pick_pod works correctly
assert pod_name == pod_name_again
logs_res = kubectl('logs', pod_name, capture_output=True)
logs = ensure_str(logs_res.stdout)
assert 'FOO=BAR' in logs
# 跑第二次只是为了看看清理过程能否顺利执行, 保证不会报错
run(lain, args=['job', '--force', 'env'])
values = load_dummy_values()
web_proc = values['deployments']['web']
web_proc.update({
'imagePullPolicy': 'Always',
'terminationGracePeriodSeconds': 1,
})
# add one extra ingress rule to values.yaml
dev_host = f'{DUMMY_APPNAME}-dev'
full_host = 'dummy.full.domain'
values['ingresses'].extend([
{
'host': dev_host,
'deployName': 'web-dev',
'paths': ['/']
},
{
'host': full_host,
'deployName': 'web',
'paths': ['/']
},
])
values['jobs'] = {'init': {'command': ['echo', 'migrate']}}
yadu(values, DUMMY_VALUES_PATH)
overrideReplicaCount = 3
overrideImageTag = 'latest'
# add another env
run(lain, args=['env', 'add', 'SCALE=BANANA'])
web_dev_proc = deepcopy(web_proc)
web_dev_proc.update({
'replicaCount': overrideReplicaCount,
'imageTag': overrideImageTag,
})
# adjust replicaCount and imageTag in override values file
override_values = {
'deployments': {
'web-dev': web_dev_proc,
},
# this is just used to ensure helm template rendering
'ingressAnnotations': {
'nginx.ingress.kubernetes.io/proxy-next-upstream-timeout': 1,
},
'externalIngresses': [
{
'host': 'dummy-public.foo.cn',
'deployName': 'web',
'paths': ['/']
},
{
'host': 'dummy-public.bar.cn',
'deployName': 'web',
'paths': ['/']
},
],
}
yadu(override_values, f'{CHART_DIR_NAME}/values-{TEST_CLUSTER}.yaml')
def get_helm_values():
ctx = context()
helm_values = ctx.obj['values']
return helm_values
# check if values-[TEST_CLUSTER].yaml currectly overrides helm context
_, helm_values = run_under_click_context(get_helm_values)
assert helm_values['deployments']['web-dev'][
'replicaCount'] == overrideReplicaCount
# deploy again to create newly added ingress rule
run(lain, args=['deploy', '--set', f'imageTag={DUMMY_IMAGE_TAG}'])
# check if the new ingress rule is created
res = kubectl(
'get',
'ing',
'-l',
f'app.kubernetes.io/name={DUMMY_APPNAME}',
'-o=jsonpath={..metadata.name}',
capture_output=True,
)
assert not res.returncode
domain = TEST_CLUSTER_CONFIG['domain']
assert set(res.stdout.decode('utf-8').split()) == {
tell_ing_name(full_host, DUMMY_APPNAME, domain, 'web'),
tell_ing_name(DUMMY_APPNAME, DUMMY_APPNAME, domain, 'web'),
f'dummy-public-foo-cn-{DUMMY_APPNAME}-web',
tell_ing_name(dev_host, DUMMY_APPNAME, domain, 'web-dev'),
f'dummy-public-bar-cn-{DUMMY_APPNAME}-web',
}
# check pod name match its corresponding deploy name
dummy_resp = url_get_json(DUMMY_URL)
assert tell_pod_deploy_name(
dummy_resp['env']['HOSTNAME']) == f'{DUMMY_APPNAME}-web'
dummy_dev_resp = url_get_json(DUMMY_DEV_URL)
assert (tell_pod_deploy_name(
dummy_dev_resp['env']['HOSTNAME']) == f'{DUMMY_APPNAME}-web-dev')
# env is overriden in dummy-dev, see default values.yaml
assert dummy_dev_resp['env']['FOO'] == 'BAR'
assert dummy_dev_resp['env']['SCALE'] == 'BANANA'
assert dummy_dev_resp['env']['LAIN_CLUSTER'] == TEST_CLUSTER
assert dummy_dev_resp['env']['K8S_NAMESPACE'] == TEST_CLUSTER_CONFIG.get(
'namespace', 'default')
assert dummy_dev_resp['env']['IMAGE_TAG'] == DUMMY_IMAGE_TAG
# check if replicaCount is correctly overriden
res = kubectl(
'get',
'deploy',
f'{DUMMY_APPNAME}-web-dev',
'-o=jsonpath={.spec.replicas}',
capture_output=True,
)
assert res.stdout.decode('utf-8').strip() == str(overrideReplicaCount)
# check if imageTag is correctly overriden
web_image = get_deploy_image(f'{DUMMY_APPNAME}-web')
assert web_image.endswith(DUMMY_IMAGE_TAG)
web_dev_image = get_deploy_image(f'{DUMMY_APPNAME}-web-dev')
assert web_dev_image.endswith(overrideImageTag)
# rollback imageTag for web-dev using `lain update_image`
run(lain, args=['update-image', 'web-dev'])
# restart a few times to test lain restart functionalities
run(lain, args=['restart', 'web', '--wait'])
run(lain, args=['restart', '--graceful', '--wait'])
dummy_dev_resp = url_get_json(DUMMY_DEV_URL)
# if dummy-dev is at the correct imageTag, that means lain update-image is
# working correctly, and lain restart too
assert dummy_dev_resp['env']['IMAGE_TAG'] == DUMMY_IMAGE_TAG
run(lain, args=['--auto-pilot', 'env', 'add', f'treasure={RANDOM_STRING}'])
dummy_resp = url_get_json(DUMMY_URL)
# --auto-pilot will trigger a graceful restart, verify by confirming the
# added env inside the freshly created containers
assert dummy_resp['env']['treasure'] == RANDOM_STRING
def test_values():
values = load_dummy_values()
domain = TEST_CLUSTER_CONFIG['domain']
values['env'] = {'SOMETHING': 'ELSE', 'OVERRIDE_BY_PROC': 'old'}
ing_anno = {'fake-annotations': 'bar'}
values['ingresses'] = [
{
'host': 'dummy',
'deployName': 'web',
'paths': ['/'],
'annotations': ing_anno
},
{
'host': f'dummy.{domain}',
'deployName': 'web',
'paths': ['/']
},
]
values['externalIngresses'] = [
{
'host': 'dummy.public.com',
'deployName': 'web',
'paths': ['/'],
'annotations': ing_anno,
},
{
'host': 'public.com',
'deployName': 'web',
'paths': ['/']
},
]
values['labels'] = {'foo': 'bar'}
web_proc = values['deployments']['web']
nodePort = 32333
fake_proc_sa = 'procsa'
web_proc.update({
'env': {
'OVERRIDE_BY_PROC': 'new'
},
'podAnnotations': {
'prometheus.io/scrape': 'true'
},
'workingDir': RANDOM_STRING,
'hostNetwork': True,
'nodePort': nodePort,
'serviceAccountName': fake_proc_sa,
'nodes': ['node-1'],
})
yadu(values, DUMMY_VALUES_PATH)
k8s_specs = render_k8s_specs()
ingresses = [spec for spec in k8s_specs if spec['kind'] == 'Ingress']
domain = TEST_CLUSTER_CONFIG['domain']
internal_ing = next(ing for ing in ingresses
if ing['metadata']['name'] == tell_ing_name(
DUMMY_APPNAME, DUMMY_APPNAME, domain, 'web'))
dic_contains(internal_ing['metadata']['annotations'], ing_anno)
dummy_public_com = next(
ing for ing in ingresses
if ing['metadata']['name'] == 'dummy-public-com-dummy-web')
dic_contains(dummy_public_com['metadata']['annotations'], ing_anno)
if 'clusterIssuer' in TEST_CLUSTER_CONFIG:
# when tls is not available, skip this test
for ing in ingresses:
spec = ing['spec']
rule = spec['rules'][0]
domain = rule['host']
tls = ing['spec']['tls']
tls_name = tls[0]['secretName']
tls_hosts = tls[0]['hosts']
assert set(tls_hosts) == set(make_wildcard_domain(domain))
assert (rule['http']['paths'][0]['backend']['service']['port']
['number'] == nodePort)
assert tls_name == tell_domain_tls_name(tls_hosts[0])
deployment = next(spec for spec in k8s_specs
if spec['kind'] == 'Deployment')
sa = deployment['spec']['template']['spec']['serviceAccountName']
assert sa == fake_proc_sa
# check if podAnnotations work
assert (deployment['spec']['template']['metadata']['annotations']
['prometheus.io/scrape'] == 'true')
container_spec = deployment['spec']['template']['spec']
assert container_spec['hostNetwork'] is True
containers = container_spec['containers'][0]
assert containers['workingDir'] == RANDOM_STRING
env_dic = {}
for pair in container_spec['containers'][0]['env']:
env_dic[pair['name']] = pair['value']
assert env_dic == {
'LAIN_CLUSTER': TEST_CLUSTER,
'K8S_NAMESPACE': TEST_CLUSTER_CONFIG.get('namespace', 'default'),
'IMAGE_TAG': 'overridden-during-deploy',
'SOMETHING': 'ELSE',
'OVERRIDE_BY_PROC': 'new',
}
assert container_spec['affinity']['nodeAffinity']
assert deployment['metadata']['labels']['foo'] == 'bar'
match_expression = container_spec['affinity']['nodeAffinity'][
'requiredDuringSchedulingIgnoredDuringExecution']['nodeSelectorTerms'][
0]['matchExpressions'][0]
assert match_expression['key'] == f'{DUMMY_APPNAME}-web'
service = next(spec for spec in k8s_specs if spec['kind'] == 'Service')
service_spec = service['spec']
port = service_spec['ports'][0]
assert port['nodePort'] == port['port'] == nodePort
assert port['targetPort'] == 5000
def render_with_override_values(dic):
yadu(dic, join(CHART_DIR_NAME, f'values-{TEST_CLUSTER}.yaml'))
k8s_specs = render_k8s_specs()
return k8s_specs