def testCombineReposOverrides(self):
repos = util.combine_repos([
"kubeflow/kubeflow@HEAD", "kubeflow/tf-operator@HEAD",
"kubeflow/kubeflow@12345", "kubeflow/tf-operator@23456"
])
expected_repos = {
"kubeflow/kubeflow": "12345",
"kubeflow/tf-operator": "23456"
}
self.assertDictEqual(repos, expected_repos)
def testCombineReposExtras(self):
repos = util.combine_repos([
"kubeflow/kubeflow@HEAD", "kubeflow/tf-operator@HEAD",
"kubeflow/kfctl@12345", "kubeflow/katib@23456"
])
expected_repos = {
"kubeflow/kubeflow": "HEAD",
"kubeflow/tf-operator": "HEAD",
"kubeflow/kfctl": "12345",
"kubeflow/katib": "23456"
}
self.assertDictEqual(repos, expected_repos)
def build(self):
self.workflow = self._build_workflow()
task_template = self._build_task_template()
py3_template = argo_build_util.deep_copy(task_template)
py3_template["container"]["image"] = "gcr.io/kubeflow-ci/test-worker-py3:e9afed1-dirty"
#**************************************************************************
# Checkout
# create the checkout step
checkout = argo_build_util.deep_copy(task_template)
# Construct the list of repos to checkout
list_of_repos = DEFAULT_REPOS
list_of_repos.append(self.main_repo)
list_of_repos.extend(self.extra_repos)
repos = util.combine_repos(list_of_repos)
repos_str = ','.join(['%s@%s' % (key, value) for (key, value) in repos.items()])
# If we are using a specific branch (e.g. periodic tests for release branch)
# then we need to use depth = all; otherwise checkout out the branch
# will fail. Otherwise we checkout with depth=30. We want more than
# depth=1 because the depth will determine our ability to find the common
# ancestor which affects our ability to determine which files have changed
depth = 30
if os.getenv("BRANCH_NAME"):
logging.info("BRANCH_NAME=%s; setting detph=all",
os.getenv("BRANCH_NAME"))
depth = "all"
checkout["name"] = "checkout"
checkout["container"]["command"] = ["/usr/local/bin/checkout_repos.sh",
"--repos=" + repos_str,
"--depth={0}".format(depth),
"--src_dir=" + self.src_root_dir]
argo_build_util.add_task_to_dag(self.workflow, E2E_DAG_NAME, checkout, [])
# Change the workfing directory for all subsequent steps
task_template["container"]["workingDir"] = os.path.join(
self.kfctl_pytest_dir)
py3_template["container"]["workingDir"] = os.path.join(self.kfctl_pytest_dir)
#**************************************************************************
# Run build_kfctl and deploy kubeflow
step_name = "kfctl-build-deploy"
command = [
"pytest",
"kfctl_go_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
"--app_name=" + self.app_name,
"--config_path=" + self.config_path,
"--values=" + self.values_str,
"--build_and_apply=" + str(self.build_and_apply),
# Increase the log level so that info level log statements show up.
# TODO(https://github.com/kubeflow/testing/issues/372): If we
# set a unique artifacts dir for each workflow with the proper
# prefix that should work.
"--log-cli-level=info",
"--junitxml=" + self.artifacts_dir + "/junit_kfctl-build-test"
+ self.config_name + ".xml",
# TODO(jlewi) Test suite name needs to be unique based on parameters.
#
"-o", "junit_suite_name=test_kfctl_go_deploy_" + self.config_name,
"--app_path=" + self.app_dir,
"--kfctl_repo_path=" + self.src_dir,
"--self_signed_cert=True",
]
dependences = [checkout["name"]]
build_kfctl = self._build_step(step_name, self.workflow, E2E_DAG_NAME,
py3_template, command, dependences)
#**************************************************************************
# Wait for Kubeflow to be ready
step_name = "kubeflow-is-ready"
command = [
"pytest",
"kf_is_ready_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
# TODO(jlewi): We should update kf_is_ready_test to take the config
# path and then based on the KfDef spec kf_is_ready_test should
# figure out what to do.
"--use_basic_auth={0}".format(self.use_basic_auth),
# TODO(jlewi): We should be using ISTIO always so can we stop
# setting this
"--use_istio=true",
# Increase the log level so that info level log statements show up.
"--log-cli-level=info",
"--junitxml=" + os.path.join(self.artifacts_dir,
"junit_kfctl-is-ready-test-" +
self.config_name + ".xml"),
# Test suite name needs to be unique based on parameters
"-o", "junit_suite_name=test_kf_is_ready_" + self.config_name,
"--app_path=" + self.app_dir,
]
dependences = [build_kfctl["name"]]
kf_is_ready = self._build_step(step_name, self.workflow, E2E_DAG_NAME, task_template,
command, dependences)
#**************************************************************************
# Wait for endpoint to be ready
if self.test_endpoint:
self._test_endpoint_step_name = "endpoint-is-ready"
command = ["pytest",
"endpoint_ready_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
# Increase the log level so that info level log statements show up.
"--log-cli-level=info",
"--junitxml=" + self.artifacts_dir + "/junit_endpoint-is-ready-test-" + self.config_name + ".xml",
# Test suite name needs to be unique based on parameters
"-o", "junit_suite_name=test_endpoint_is_ready_" + self.config_name,
"--app_path=" + self.app_dir,
"--app_name=" + self.app_name,
"--use_basic_auth={0}".format(self.use_basic_auth),
]
dependencies = [build_kfctl["name"]]
endpoint_ready = self._build_step(self._test_endpoint_step_name,
self.workflow, E2E_DAG_NAME, py3_template,
command, dependencies)
self._test_endpoint_template_name = endpoint_ready["name"]
#**************************************************************************
# Do kfctl apply again. This test will be skip if it's presubmit.
step_name = "kfctl-second-apply"
command = [
"pytest",
"kfctl_second_apply.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
"--log-cli-level=info",
"--junitxml=" + os.path.join(self.artifacts_dir,
"junit_kfctl-second-apply-test-" +
self.config_name + ".xml"),
# Test suite name needs to be unique based on parameters
"-o", "junit_suite_name=test_kfctl_second_apply_" + self.config_name,
"--app_path=" + self.app_dir,
"--kfctl_path=" + self.kfctl_path,
]
if self.test_endpoint:
dependences = [kf_is_ready["name"], endpoint_ready["name"]]
else:
dependences = [kf_is_ready["name"]]
kf_second_apply = self._build_step(step_name, self.workflow, E2E_DAG_NAME, task_template,
command, dependences)
self._build_tests_dag()
# Add a task to run the dag
dependencies = [kf_is_ready["name"]]
self._run_tests_step_name = TESTS_DAG_NAME
run_tests_template_name = TESTS_DAG_NAME
argo_build_util.add_task_only_to_dag(self.workflow, E2E_DAG_NAME, self._run_tests_step_name,
run_tests_template_name,
dependencies)
#***************************************************************************
# create_pr_symlink
#***************************************************************************
# TODO(jlewi): run_e2e_workflow.py should probably create the PR symlink
step_name = "create-pr-symlink"
command = ["python",
"-m",
"kubeflow.testing.prow_artifacts",
"--artifacts_dir=" + self.output_dir,
"create_pr_symlink"]
if self.bucket:
command.append(self.bucket)
dependences = [checkout["name"]]
symlink = self._build_step(step_name, self.workflow, E2E_DAG_NAME, task_template,
command, dependences)
self._build_exit_dag()
# Set the labels on all templates
self.workflow = argo_build_util.set_task_template_labels(self.workflow)
return self.workflow
def build(self):
self.workflow = self._build_workflow()
task_template = self._build_task_template()
py3_template = argo_build_util.deep_copy(task_template)
py3_template["container"][
"image"] = "527798164940.dkr.ecr.us-west-2.amazonaws.com/aws-kubeflow-ci/test-worker:v1.2-branch"
default_namespace = "kubeflow"
#**************************************************************************
# Checkout
# create the checkout step
checkout = argo_build_util.deep_copy(task_template)
# Construct the list of repos to checkout
list_of_repos = DEFAULT_REPOS
list_of_repos.append(self.main_repo)
list_of_repos.extend(self.extra_repos)
repos = util.combine_repos(list_of_repos)
repos_str = ','.join(
['%s@%s' % (key, value) for (key, value) in repos.items()])
# If we are using a specific branch (e.g. periodic tests for release branch)
# then we need to use depth = all; otherwise checkout out the branch
# will fail. Otherwise we checkout with depth=30. We want more than
# depth=1 because the depth will determine our ability to find the common
# ancestor which affects our ability to determine which files have changed
depth = 30
if os.getenv("BRANCH_NAME"):
logging.info("BRANCH_NAME=%s; setting detph=all",
os.getenv("BRANCH_NAME"))
depth = "all"
checkout["name"] = "checkout"
checkout["container"]["command"] = [
"/usr/local/bin/checkout_repos.sh", "--repos=" + repos_str,
"--depth={0}".format(depth), "--src_dir=" + self.src_root_dir
]
argo_build_util.add_task_to_dag(self.workflow, E2E_DAG_NAME, checkout,
[])
# Change the working directory for all subsequent steps
task_template["container"]["workingDir"] = os.path.join(
self.kfctl_pytest_dir)
py3_template["container"]["workingDir"] = os.path.join(
self.kfctl_pytest_dir)
#***************************************************************************
# create_pr_symlink
#***************************************************************************
# TODO(jlewi): run_e2e_workflow.py should probably create the PR symlink
step_name = "create-pr-symlink"
command = [
"python", "-m",
"kubeflow.testing.cloudprovider.aws.prow_artifacts",
"--artifacts_dir=" + self.output_dir, "create_pr_symlink_s3",
"--bucket=" + self.bucket
]
dependences = [checkout["name"]]
symlink = self._build_step(step_name, self.workflow, E2E_DAG_NAME,
task_template, command, dependences)
#**************************************************************************
# Run build_kfctl
step_name = "kfctl-build-deploy"
command = [
"pytest",
"kfctl_go_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
"--config_path=" + self.config_path,
"--values=" + self.values_str,
# Increase the log level so that info level log statements show up.
# TODO(https://github.com/kubeflow/testing/issues/372): If we
# set a unique artifacts dir for each workflow with the proper
# prefix that should work.
"--log-cli-level=info",
"--junitxml=" + self.artifacts_dir + "/junit_kfctl-build-test" +
self.config_name + ".xml",
# TODO(jlewi) Test suite name needs to be unique based on parameters.
"-o",
"junit_suite_name=test_kfctl_go_deploy_" + self.config_name,
"--kfctl_repo_path=" + self.src_dir,
]
dependences = [checkout["name"]]
build_kfctl = self._build_step(step_name, self.workflow, E2E_DAG_NAME,
py3_template, command, dependences)
#**************************************************************************
# Create EKS cluster for E2E test
step_name = "kfctl-create-cluster"
command = [
"pytest",
"kfctl_create_cluster_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
"--cluster_name=" + self.cluster_name,
"--eks_cluster_version=" + str(self.eks_cluster_version),
# Embedded Script in the ECR Image
"--cluster_creation_script=" +
"/usr/local/bin/create-eks-cluster.sh",
"--values=" + self.values_str,
# Increase the log level so that info level log statements show up.
# TODO(https://github.com/kubeflow/testing/issues/372): If we
# set a unique artifacts dir for each workflow with the proper
# prefix that should work.
"--log-cli-level=info",
"--junitxml=" + self.artifacts_dir + "/junit_kfctl-build-test" +
self.config_name + ".xml",
# TODO(jlewi) Test suite name needs to be unique based on parameters.
"-o",
"junit_suite_name=test_kfctl_go_deploy_" + self.config_name,
]
dependences = [checkout["name"]]
create_cluster = self._build_step(step_name, self.workflow,
E2E_DAG_NAME, py3_template, command,
dependences)
#**************************************************************************
# Deploy Kubeflow
step_name = "kfctl-deploy-kubeflow"
command = [
"pytest",
"kfctl_deploy_kubeflow_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
"--cluster_name=" + self.cluster_name,
# Embedded Script in the ECR Image
"--cluster_creation_script=" +
"/usr/local/bin/create-eks-cluster.sh",
"--config_path=" + self.config_path,
"--values=" + self.values_str,
"--build_and_apply=" + str(self.build_and_apply),
# Increase the log level so that info level log statements show up.
# TODO(https://github.com/kubeflow/testing/issues/372): If we
# set a unique artifacts dir for each workflow with the proper
# prefix that should work.
"--log-cli-level=info",
"--junitxml=" + self.artifacts_dir + "/junit_kfctl-build-test" +
self.config_name + ".xml",
# TODO(jlewi) Test suite name needs to be unique based on parameters.
"-o",
"junit_suite_name=test_kfctl_go_deploy_" + self.config_name,
"--app_path=" + self.app_dir,
"--kfctl_repo_path=" + self.src_dir,
]
dependences = [
build_kfctl["name"], create_cluster["name"], symlink["name"]
]
deploy_kf = self._build_step(step_name, self.workflow, E2E_DAG_NAME,
py3_template, command, dependences)
#**************************************************************************
# Wait for Kubeflow to be ready
step_name = "kubeflow-is-ready"
command = [
"pytest",
"kf_is_ready_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
# TODO(jlewi): We should update kf_is_ready_test to take the config
# path and then based on the KfDef spec kf_is_ready_test should
# figure out what to do.
"--use_basic_auth={0}".format(self.use_basic_auth),
# Increase the log level so that info level log statements show up.
"--log-cli-level=info",
"--junitxml=" + os.path.join(
self.artifacts_dir,
"junit_kfctl-is-ready-test-" + self.config_name + ".xml"),
# Test suite name needs to be unique based on parameters
"-o",
"junit_suite_name=test_kf_is_ready_" + self.config_name,
"--app_path=" + self.app_dir,
"--cluster_name=" + self.cluster_name,
"--namespace=" + default_namespace,
]
dependences = [deploy_kf["name"]]
kf_is_ready = self._build_step(step_name, self.workflow, E2E_DAG_NAME,
task_template, command, dependences)
#**************************************************************************
# Run functional tests
dependences = [kf_is_ready["name"]]
dependences = self._build_tests_dag(dependences=dependences)
#***********************************************************************
# Delete Kubeflow
# Putting Delete Kubeflow here is deletion functionality should be tested out of exit DAG
step_name = "kfctl-delete-wrong-host"
command = [
"pytest",
"kfctl_delete_wrong_cluster.py",
"-s",
"--log-cli-level=info",
"--timeout=1000",
"--junitxml=" + self.artifacts_dir +
"/junit_kfctl-go-delete-wrong-cluster-test.xml",
"--app_path=" + self.app_dir,
"--kfctl_path=" + self.kfctl_path,
"--cluster_name=" + self.cluster_name,
]
kfctl_delete_wrong_cluster = self._build_step(step_name, self.workflow,
E2E_DAG_NAME,
task_template, command,
dependences)
kfctl_delete_wrong_cluster["container"][
"workingDir"] = self.kfctl_pytest_dir
step_name = "kfctl-delete"
command = [
"pytest",
"kfctl_delete_test.py",
"-s",
"--log-cli-level=info",
"--timeout=1000",
"--junitxml=" + self.artifacts_dir +
"/junit_kfctl-go-delete-test.xml",
"--app_path=" + self.app_dir,
"--kfctl_path=" + self.kfctl_path,
"--cluster_name=" + self.cluster_name,
]
kfctl_delete = self._build_step(step_name, self.workflow, E2E_DAG_NAME,
task_template, command,
["kfctl-delete-wrong-host"])
kfctl_delete["container"]["workingDir"] = self.kfctl_pytest_dir
#***************************************************************************
# Exit DAG
#***************************************************************************
self._build_exit_dag()
# Set the labels on all templates
self.workflow = argo_build_util.set_task_template_labels(self.workflow)
return self.workflow
def build(self):
self.workflow = self._build_workflow()
task_template = self._build_task_template()
#**************************************************************************
# Checkout
# create the checkout step
checkout = argo_build_util.deep_copy(task_template)
# Construct the list of repos to checkout
list_of_repos = DEFAULT_REPOS
list_of_repos.append(self.main_repo)
list_of_repos.extend(self.extra_repos)
repos = util.combine_repos(list_of_repos)
repos_str = ','.join(
['%s@%s' % (key, value) for (key, value) in repos.items()])
checkout["name"] = "checkout"
checkout["container"]["command"] = [
"/usr/local/bin/checkout_repos.sh", "--repos=" + repos_str,
"--src_dir=" + self.src_root_dir
]
argo_build_util.add_task_to_dag(self.workflow, E2E_DAG_NAME, checkout,
[])
# Change the workfing directory for all subsequent steps
task_template["container"]["workingDir"] = os.path.join(
self.kfctl_pytest_dir)
#**************************************************************************
# Run build_kfctl and deploy kubeflow
step_name = "kfctl-build-deploy"
command = [
"pytest",
"kfctl_go_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
"--config_path=" + self.config_path,
"--build_and_apply=" + str(self.build_and_apply),
# Increase the log level so that info level log statements show up.
# TODO(https://github.com/kubeflow/testing/issues/372): If we
# set a unique artifacts dir for each workflow with the proper
# prefix that should work.
"--log-cli-level=info",
"--junitxml=" + self.artifacts_dir + "/junit_kfctl-build-test" +
self.config_name + ".xml",
# TODO(jlewi) Test suite name needs to be unique based on parameters.
#
"-o",
"junit_suite_name=test_kfctl_go_deploy_" + self.config_name,
"--app_path=" + self.app_dir,
]
dependences = [checkout["name"]]
build_kfctl = self._build_step(step_name, self.workflow, E2E_DAG_NAME,
task_template, command, dependences)
#**************************************************************************
# Wait for Kubeflow to be ready
step_name = "kubeflow-is-ready"
command = [
"pytest",
"kf_is_ready_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
# TODO(jlewi): We should update kf_is_ready_test to take the config
# path and then based on the KfDef spec kf_is_ready_test should
# figure out what to do.
"--use_basic_auth={0}".format(self.use_basic_auth),
# TODO(jlewi): We should be using ISTIO always so can we stop
# setting this
"--use_istio=true",
# Increase the log level so that info level log statements show up.
"--log-cli-level=info",
"--junitxml=" + os.path.join(
self.artifacts_dir,
"junit_kfctl-is-ready-test-" + self.config_name + ".xml"),
# Test suite name needs to be unique based on parameters
"-o",
"junit_suite_name=test_kf_is_ready_" + self.config_name,
"--app_path=" + self.app_dir,
]
dependences = [build_kfctl["name"]]
kf_is_ready = self._build_step(step_name, self.workflow, E2E_DAG_NAME,
task_template, command, dependences)
#**************************************************************************
# Wait for endpoint to be ready
if self.test_endpoint:
step_name = "endpoint-is-ready"
command = [
"pytest",
"endpoint_ready_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
# Increase the log level so that info level log statements show up.
"--log-cli-level=info",
# Test timeout in seconds.
"--timeout=1800",
"--junitxml=" + self.artifacts_dir +
"/junit_endpoint-is-ready-test-" + self.config_name + ".xml",
# Test suite name needs to be unique based on parameters
"-o",
"junit_suite_name=test_endpoint_is_ready_" + self.config_name,
"--app_path=" + self.app_dir,
"--app_name=" + self.app_name,
"--use_basic_auth={0}".format(self.use_basic_auth),
]
dependencies = [build_kfctl["name"]]
endpoint_ready = self._build_step(step_name, self.workflow,
E2E_DAG_NAME, task_template,
command, dependencies)
self._build_tests_dag()
# Add a task to run the dag
dependencies = [kf_is_ready["name"]]
argo_build_util.add_task_only_to_dag(self.workflow, E2E_DAG_NAME,
TESTS_DAG_NAME, TESTS_DAG_NAME,
dependencies)
#***************************************************************************
# create_pr_symlink
#***************************************************************************
# TODO(jlewi): run_e2e_workflow.py should probably create the PR symlink
step_name = "create-pr-symlink"
command = [
"python", "-m", "kubeflow.testing.prow_artifacts",
"--artifacts_dir=" + self.output_dir, "create_pr_symlink"
]
if self.bucket:
command.append(self.bucket)
dependences = [checkout["name"]]
symlink = self._build_step(step_name, self.workflow, E2E_DAG_NAME,
task_template, command, dependences)
self._build_exit_dag()
# Set the labels on all templates
self.workflow = argo_build_util.set_task_template_labels(self.workflow)
return self.workflow
def testCombineReposDefault(self):
repos = util.combine_repos([])
expected_repos = {}
self.assertDictEqual(repos, expected_repos)
