def build(self):
"""Build the Argo workflow graph"""
workflow = self.build_init_workflow(exit_dag=False)
task_template = self.build_task_template()
# build manifests with kustomize
kustomize_build_task = self._kustomize_build_task(task_template)
argo_build_util.add_task_to_dag(workflow, workflow_utils.E2E_DAG_NAME,
kustomize_build_task,
[self.mkdir_task_name])
# Test building Notebook Controller image using Kaniko
dockerfile = ("%s/components/notebook-controller"
"/Dockerfile") % self.src_dir
context = "dir://%s/components/" % self.src_dir
destination = "notebok-controller-test"
kaniko_task = self.create_kaniko_task(task_template,
dockerfile,
context,
destination,
no_push=True)
argo_build_util.add_task_to_dag(workflow, workflow_utils.E2E_DAG_NAME,
kaniko_task, [self.mkdir_task_name])
# Set the labels on all templates
workflow = argo_build_util.set_task_template_labels(workflow)
return workflow
def build(self):
"""Build the Argo workflow graph"""
workflow = self.build_init_workflow(exit_dag=False)
task_template = self.build_task_template()
# Test building notebook-server-jupyter-pytorch images using Kaniko
dockerfile = ("%s/components/example-notebook-servers"
"/jupyter-pytorch/cpu.Dockerfile") % self.src_dir
context = "dir://%s/components/example-notebook-servers/jupyter-pytorch/" % self.src_dir
destination = "notebook-server-jupyter-pytorch-cpu-test"
kaniko_task = self.create_kaniko_task(task_template, dockerfile,
context, destination, no_push=True)
argo_build_util.add_task_to_dag(workflow,
workflow_utils.E2E_DAG_NAME,
kaniko_task, [self.mkdir_task_name])
dockerfile_cuda = ("%s/components/example-notebook-servers"
"/jupyter-pytorch/cuda.Dockerfile") % self.src_dir
destination_cuda = "notebook-server-jupyter-pytorch-cuda-test"
kaniko_task_cuda = self.create_kaniko_task(task_template, dockerfile_cuda,
context, destination_cuda, no_push=True)
argo_build_util.add_task_to_dag(workflow,
workflow_utils.E2E_DAG_NAME,
kaniko_task_cuda, [self.mkdir_task_name])
# Set the labels on all templates
workflow = argo_build_util.set_task_template_labels(workflow)
return workflow
def build(self):
"""Build the Argo workflow graph"""
workflow = self.build_init_workflow(exit_dag=False)
task_template = self.build_task_template()
app_dir = "%s/components/crud-web-apps/volumes" % self.src_dir
# build manifests with kustomize
kustomize_build_task = self._kustomize_build_task(task_template)
argo_build_util.add_task_to_dag(workflow, workflow_utils.E2E_DAG_NAME,
kustomize_build_task,
[self.mkdir_task_name])
# Test building VWA image using Kaniko
dockerfile = "%s/Dockerfile" % app_dir
context = "dir://%s/components/crud-web-apps" % self.src_dir
destination = "vwa-test"
kaniko_task = self.create_kaniko_task(task_template, dockerfile,
context, destination,
no_push=True)
argo_build_util.add_task_to_dag(workflow,
workflow_utils.E2E_DAG_NAME,
kaniko_task, [self.mkdir_task_name])
# install npm modules
ui_dir = "%s/frontend" % app_dir
modules_install_task = self.create_install_modules_task(task_template,
ui_dir)
argo_build_util.add_task_to_dag(workflow, workflow_utils.E2E_DAG_NAME,
modules_install_task,
[self.mkdir_task_name])
# check if the frontend code is properly formatted
format_typescript = self.create_format_typescript_task(task_template,
ui_dir)
argo_build_util.add_task_to_dag(workflow,
workflow_utils.E2E_DAG_NAME,
format_typescript,
[modules_install_task["name"]])
# check if the backend python code is properly formatted
backend_dir = "%s/backend" % app_dir
format_python = self.create_format_python_task(task_template,
backend_dir)
argo_build_util.add_task_to_dag(workflow,
workflow_utils.E2E_DAG_NAME,
format_python,
[self.mkdir_task_name])
# Set the labels on all templates
workflow = argo_build_util.set_task_template_labels(workflow)
return workflow
def build(self):
"""Build the Argo workflow graph"""
workflow = self.build_init_workflow()
task_template = self.build_task_template()
# install npm modules
modules_install_task = self._create_install_modules_task(task_template)
argo_build_util.add_task_to_dag(workflow, workflow_utils.E2E_DAG_NAME,
modules_install_task,
[self.mkdir_task_name])
# run common ui frontend tests
ui_tests_task = self._create_ui_tests_task(task_template)
argo_build_util.add_task_to_dag(workflow, workflow_utils.E2E_DAG_NAME,
ui_tests_task,
[modules_install_task["name"]])
# build the node module from the lib source code
build_step = self._create_ui_build_task(task_template)
argo_build_util.add_task_to_dag(workflow, workflow_utils.E2E_DAG_NAME,
build_step,
[modules_install_task["name"]])
# EXIT-HANDLER: remove node_modules folder as exit handler
rm_node_modules = self._create_exit_handler(task_template)
argo_build_util.add_task_to_dag(workflow, workflow_utils.EXIT_DAG_NAME,
rm_node_modules, [])
# Set the labels on all templates
workflow = argo_build_util.set_task_template_labels(workflow)
return workflow
def build_init_workflow(self):
"""Build the Argo workflow graph"""
workflow = self._build_workflow()
task_template = self.build_task_template()
# checkout the code
checkout_task = self._create_checkout_task(task_template)
argo_build_util.add_task_to_dag(workflow, E2E_DAG_NAME, checkout_task,
[])
# create the artifacts directory
mkdir_task = self._create_make_dir_task(task_template)
argo_build_util.add_task_to_dag(workflow, E2E_DAG_NAME, mkdir_task,
[checkout_task["name"]])
return workflow
def _build_step(self, name, workflow, dag_name, task_template, command,
dependencies):
"""Syntactic sugar to add a step to the workflow"""
step = argo_build_util.deep_copy(task_template)
step["name"] = name
step["container"]["command"] = command
argo_build_util.add_task_to_dag(workflow, dag_name, step, dependencies)
# Return the newly created template; add_task_to_dag makes a copy of the template
# So we need to fetch it from the workflow spec.
for t in workflow["spec"]["templates"]:
if t["name"] == name:
return t
return None
def _build_step(self, name, workflow, dag_name, task_template,
command, dependences):
"""Syntactic sugar to add a step to the workflow"""
step = argo_build_util.deep_copy(task_template)
step["name"] = name
step["container"]["command"] = command
return argo_build_util.add_task_to_dag(workflow, dag_name, step, dependences)
def build(self, dockerfile, context, destination):
"""Build the Argo workflow graph"""
workflow = self.build_init_workflow(exit_dag=False)
task_template = self.build_task_template()
# Build component OCI image using Kaniko
dockerfile = ("%s/%s") % (self.src_dir, dockerfile)
context = "dir://%s/%s" % (self.src_dir, context)
destination = destination
kaniko_task = self.create_kaniko_task(task_template, dockerfile,
context, destination)
argo_build_util.add_task_to_dag(workflow,
ci.workflow_utils.E2E_DAG_NAME,
kaniko_task, [self.mkdir_task_name])
# Set the labels on all templates
workflow = argo_build_util.set_task_template_labels(workflow)
return workflow
def build(self):
"""Build the Argo workflow graph"""
workflow = self.build_init_workflow(exit_dag=False)
task_template = self.build_task_template()
# Build JWA using Kaniko
dockerfile = ("%s/components/crud-web-apps"
"/jupyter/Dockerfile") % self.src_dir
context = "dir://%s/components/crud-web-apps" % self.src_dir
destination = config.JUPYTER_WEB_APP_IMAGE
kaniko_task = self.create_kaniko_task(task_template, dockerfile,
context, destination)
argo_build_util.add_task_to_dag(workflow,
ci.workflow_utils.E2E_DAG_NAME,
kaniko_task, [self.mkdir_task_name])
# Set the labels on all templates
workflow = argo_build_util.set_task_template_labels(workflow)
return workflow
def build(self):
"""Build the Argo workflow graph"""
workflow = self.build_init_workflow(exit_dag=False)
task_template = self.build_task_template()
# Test building TWA image using Kaniko
dockerfile = ("%s/components/crud-web-apps"
"/tensorboards/Dockerfile") % self.src_dir
context = "dir://%s/components/crud-web-apps" % self.src_dir
destination = "twa-test"
kaniko_task = self.create_kaniko_task(task_template, dockerfile,
context, destination, no_push=True)
argo_build_util.add_task_to_dag(workflow,
workflow_utils.E2E_DAG_NAME,
kaniko_task, [self.mkdir_task_name])
# Set the labels on all templates
workflow = argo_build_util.set_task_template_labels(workflow)
return workflow
def build(self):
"""Build the Argo workflow graph"""
workflow = self.build_init_workflow()
task_template = self.build_task_template()
# install npm modules
modules_install_task = self._create_install_modules_task(task_template)
argo_build_util.add_task_to_dag(workflow, workflow_utils.E2E_DAG_NAME,
modules_install_task,
[self.mkdir_task_name])
# run common ui frontend tests
ui_tests_task = self._create_ui_tests_task(task_template)
argo_build_util.add_task_to_dag(workflow, workflow_utils.E2E_DAG_NAME,
ui_tests_task,
[modules_install_task["name"]])
# run common ui frontend tests
build_step = self._create_ui_build_task(task_template)
argo_build_util.add_task_to_dag(workflow, workflow_utils.E2E_DAG_NAME,
build_step,
[modules_install_task["name"]])
# Set the labels on all templates
workflow = argo_build_util.set_task_template_labels(workflow)
return workflow
def build(self):
"""Build the Argo workflow graph"""
workflow = self.build_init_workflow(exit_dag=False)
task_template = self.build_task_template(mem_override="6Gi")
# Build Access Management using Kaniko
dockerfile = ("%s/components/access-management"
"/Dockerfile") % self.src_dir
context = "dir://%s/components/access-management/" % self.src_dir
destination = "access-management-test"
kaniko_task = self.create_kaniko_task(task_template,
dockerfile,
context,
destination,
no_push=True)
argo_build_util.add_task_to_dag(workflow, workflow_utils.E2E_DAG_NAME,
kaniko_task, [self.mkdir_task_name])
# Set the labels on all templates
workflow = argo_build_util.set_task_template_labels(workflow)
return workflow
def build(self):
self.workflow = self._build_workflow()
task_template = self._build_task_template()
#**************************************************************************
# Checkout
# create the checkout step
main_repo = argo_build_util.get_repo_from_prow_env()
if not main_repo:
logging.info("Prow environment variables for repo not set")
main_repo = MAIN_REPO + "@HEAD"
logging.info("Main repository: %s", main_repo)
repos = [main_repo]
repos.extend(EXTRA_REPOS)
checkout = argo_build_util.deep_copy(task_template)
checkout["name"] = "checkout"
checkout["container"]["command"] = [
"/usr/local/bin/checkout_repos.sh", "--repos=" + ",".join(repos),
"--src_dir=" + self.src_root_dir
]
argo_build_util.add_task_to_dag(self.workflow, E2E_DAG_NAME, checkout,
[])
# Change the workfing directory for all subsequent steps
task_template["container"]["workingDir"] = os.path.join(
self.kfctl_pytest_dir)
#**************************************************************************
# Run build_kfctl and deploy kubeflow
step_name = "kfctl-build-deploy"
command = [
"pytest",
"kfctl_go_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
"--config_path=" + self.config_path,
"--build_and_apply=" + str(self.build_and_apply),
# Increase the log level so that info level log statements show up.
# TODO(https://github.com/kubeflow/testing/issues/372): If we
# set a unique artifacts dir for each workflow with the proper
# prefix that should work.
"--log-cli-level=info",
"--junitxml=" + self.artifacts_dir + "/junit_kfctl-build-test" +
self.config_name + ".xml",
# TODO(jlewi) Test suite name needs to be unique based on parameters.
#
"-o",
"junit_suite_name=test_kfctl_go_deploy_" + self.config_name,
"--app_path=" + self.app_dir,
]
dependences = [checkout["name"]]
build_kfctl = self._build_step(step_name, self.workflow, E2E_DAG_NAME,
task_template, command, dependences)
#**************************************************************************
# Wait for Kubeflow to be ready
step_name = "kubeflow-is-ready"
command = [
"pytest",
"kf_is_ready_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
# TODO(jlewi): We should update kf_is_ready_test to take the config
# path and then based on the KfDef spec kf_is_ready_test should
# figure out what to do.
"--use_basic_auth={0}".format(self.use_basic_auth),
# TODO(jlewi): We should be using ISTIO always so can we stop
# setting this
"--use_istio=true",
# Increase the log level so that info level log statements show up.
"--log-cli-level=info",
"--junitxml=" + os.path.join(
self.artifacts_dir,
"junit_kfctl-is-ready-test-" + self.config_name + ".xml"),
# Test suite name needs to be unique based on parameters
"-o",
"junit_suite_name=test_kf_is_ready_" + self.config_name,
"--app_path=" + self.app_dir,
]
dependences = [build_kfctl["name"]]
kf_is_ready = self._build_step(step_name, self.workflow, E2E_DAG_NAME,
task_template, command, dependences)
#**************************************************************************
# Wait for endpoint to be ready
if self.test_endpoint:
step_name = "endpoint-is-ready"
command = [
"pytest",
"endpoint_ready_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
# Increase the log level so that info level log statements show up.
"--log-cli-level=info",
# Test timeout in seconds.
"--timeout=1800",
"--junitxml=" + self.artifacts_dir +
"/junit_endpoint-is-ready-test-" + self.config_name + ".xml",
# Test suite name needs to be unique based on parameters
"-o",
"junit_suite_name=test_endpoint_is_ready_" + self.config_name,
"--app_path=" + self.app_dir,
"--app_name=" + self.app_name,
]
dependencies = [build_kfctl["name"]]
endpoint_ready = self._build_step(step_name, self.workflow,
E2E_DAG_NAME, task_template,
command, dependencies)
self._build_tests_dag()
# Add a task to run the dag
dependencies = [kf_is_ready["name"]]
argo_build_util.add_task_only_to_dag(self.workflow, E2E_DAG_NAME,
TESTS_DAG_NAME, TESTS_DAG_NAME,
dependencies)
#***************************************************************************
# create_pr_symlink
#***************************************************************************
# TODO(jlewi): run_e2e_workflow.py should probably create the PR symlink
step_name = "create-pr-symlink"
command = [
"python",
"-m",
"kubeflow.testing.prow_artifacts",
"--artifacts_dir=" + self.output_dir,
"create_pr_symlink",
"--bucket=" + self.bucket,
]
dependences = [checkout["name"]]
symlink = self._build_step(step_name, self.workflow, E2E_DAG_NAME,
task_template, command, dependences)
self._build_exit_dag()
# Set the labels on all templates
self.workflow = argo_build_util.set_task_template_labels(self.workflow)
return self.workflow
def build(self):
self.workflow = self._build_workflow()
task_template = self._build_task_template()
py3_template = argo_build_util.deep_copy(task_template)
py3_template["container"]["image"] = "gcr.io/kubeflow-ci/test-worker-py3:e9afed1-dirty"
#**************************************************************************
# Checkout
# create the checkout step
checkout = argo_build_util.deep_copy(task_template)
# Construct the list of repos to checkout
list_of_repos = DEFAULT_REPOS
list_of_repos.append(self.main_repo)
list_of_repos.extend(self.extra_repos)
repos = util.combine_repos(list_of_repos)
repos_str = ','.join(['%s@%s' % (key, value) for (key, value) in repos.items()])
# If we are using a specific branch (e.g. periodic tests for release branch)
# then we need to use depth = all; otherwise checkout out the branch
# will fail. Otherwise we checkout with depth=30. We want more than
# depth=1 because the depth will determine our ability to find the common
# ancestor which affects our ability to determine which files have changed
depth = 30
if os.getenv("BRANCH_NAME"):
logging.info("BRANCH_NAME=%s; setting detph=all",
os.getenv("BRANCH_NAME"))
depth = "all"
checkout["name"] = "checkout"
checkout["container"]["command"] = ["/usr/local/bin/checkout_repos.sh",
"--repos=" + repos_str,
"--depth={0}".format(depth),
"--src_dir=" + self.src_root_dir]
argo_build_util.add_task_to_dag(self.workflow, E2E_DAG_NAME, checkout, [])
# Change the workfing directory for all subsequent steps
task_template["container"]["workingDir"] = os.path.join(
self.kfctl_pytest_dir)
py3_template["container"]["workingDir"] = os.path.join(self.kfctl_pytest_dir)
#**************************************************************************
# Run build_kfctl and deploy kubeflow
step_name = "kfctl-build-deploy"
command = [
"pytest",
"kfctl_go_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
"--app_name=" + self.app_name,
"--config_path=" + self.config_path,
"--values=" + self.values_str,
"--build_and_apply=" + str(self.build_and_apply),
# Increase the log level so that info level log statements show up.
# TODO(https://github.com/kubeflow/testing/issues/372): If we
# set a unique artifacts dir for each workflow with the proper
# prefix that should work.
"--log-cli-level=info",
"--junitxml=" + self.artifacts_dir + "/junit_kfctl-build-test"
+ self.config_name + ".xml",
# TODO(jlewi) Test suite name needs to be unique based on parameters.
#
"-o", "junit_suite_name=test_kfctl_go_deploy_" + self.config_name,
"--app_path=" + self.app_dir,
"--kfctl_repo_path=" + self.src_dir,
"--self_signed_cert=True",
]
dependences = [checkout["name"]]
build_kfctl = self._build_step(step_name, self.workflow, E2E_DAG_NAME,
py3_template, command, dependences)
#**************************************************************************
# Wait for Kubeflow to be ready
step_name = "kubeflow-is-ready"
command = [
"pytest",
"kf_is_ready_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
# TODO(jlewi): We should update kf_is_ready_test to take the config
# path and then based on the KfDef spec kf_is_ready_test should
# figure out what to do.
"--use_basic_auth={0}".format(self.use_basic_auth),
# TODO(jlewi): We should be using ISTIO always so can we stop
# setting this
"--use_istio=true",
# Increase the log level so that info level log statements show up.
"--log-cli-level=info",
"--junitxml=" + os.path.join(self.artifacts_dir,
"junit_kfctl-is-ready-test-" +
self.config_name + ".xml"),
# Test suite name needs to be unique based on parameters
"-o", "junit_suite_name=test_kf_is_ready_" + self.config_name,
"--app_path=" + self.app_dir,
]
dependences = [build_kfctl["name"]]
kf_is_ready = self._build_step(step_name, self.workflow, E2E_DAG_NAME, task_template,
command, dependences)
#**************************************************************************
# Wait for endpoint to be ready
if self.test_endpoint:
self._test_endpoint_step_name = "endpoint-is-ready"
command = ["pytest",
"endpoint_ready_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
# Increase the log level so that info level log statements show up.
"--log-cli-level=info",
"--junitxml=" + self.artifacts_dir + "/junit_endpoint-is-ready-test-" + self.config_name + ".xml",
# Test suite name needs to be unique based on parameters
"-o", "junit_suite_name=test_endpoint_is_ready_" + self.config_name,
"--app_path=" + self.app_dir,
"--app_name=" + self.app_name,
"--use_basic_auth={0}".format(self.use_basic_auth),
]
dependencies = [build_kfctl["name"]]
endpoint_ready = self._build_step(self._test_endpoint_step_name,
self.workflow, E2E_DAG_NAME, py3_template,
command, dependencies)
self._test_endpoint_template_name = endpoint_ready["name"]
#**************************************************************************
# Do kfctl apply again. This test will be skip if it's presubmit.
step_name = "kfctl-second-apply"
command = [
"pytest",
"kfctl_second_apply.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
"--log-cli-level=info",
"--junitxml=" + os.path.join(self.artifacts_dir,
"junit_kfctl-second-apply-test-" +
self.config_name + ".xml"),
# Test suite name needs to be unique based on parameters
"-o", "junit_suite_name=test_kfctl_second_apply_" + self.config_name,
"--app_path=" + self.app_dir,
"--kfctl_path=" + self.kfctl_path,
]
if self.test_endpoint:
dependences = [kf_is_ready["name"], endpoint_ready["name"]]
else:
dependences = [kf_is_ready["name"]]
kf_second_apply = self._build_step(step_name, self.workflow, E2E_DAG_NAME, task_template,
command, dependences)
self._build_tests_dag()
# Add a task to run the dag
dependencies = [kf_is_ready["name"]]
self._run_tests_step_name = TESTS_DAG_NAME
run_tests_template_name = TESTS_DAG_NAME
argo_build_util.add_task_only_to_dag(self.workflow, E2E_DAG_NAME, self._run_tests_step_name,
run_tests_template_name,
dependencies)
#***************************************************************************
# create_pr_symlink
#***************************************************************************
# TODO(jlewi): run_e2e_workflow.py should probably create the PR symlink
step_name = "create-pr-symlink"
command = ["python",
"-m",
"kubeflow.testing.prow_artifacts",
"--artifacts_dir=" + self.output_dir,
"create_pr_symlink"]
if self.bucket:
command.append(self.bucket)
dependences = [checkout["name"]]
symlink = self._build_step(step_name, self.workflow, E2E_DAG_NAME, task_template,
command, dependences)
self._build_exit_dag()
# Set the labels on all templates
self.workflow = argo_build_util.set_task_template_labels(self.workflow)
return self.workflow
def build(self):
workflow = self._build_workflow()
task_template = self._build_task_template()
#**************************************************************************
# Checkout
# create the checkout step
main_repo = argo_build_util.get_repo_from_prow_env()
if not main_repo:
logging.info("Prow environment variables for repo not set")
main_repo = "kubeflow/testing@HEAD"
logging.info("Main repository: %s", main_repo)
repos = [main_repo]
checkout = argo_build_util.deep_copy(task_template)
checkout["name"] = "checkout"
checkout["container"]["command"] = ["/usr/local/bin/checkout_repos.sh",
"--repos=" + ",".join(repos),
"--src_dir=" + self.src_root_dir]
argo_build_util.add_task_to_dag(workflow, E2E_DAG_NAME, checkout, [])
#**************************************************************************
# Make dir
# pytest was failing trying to call makedirs. My suspicion is its
# because the two steps ended up trying to create the directory at the
# same time and classing. So we create a separate step to do it.
mkdir_step = argo_build_util.deep_copy(task_template)
mkdir_step["name"] = "make-artifacts-dir"
mkdir_step["container"]["command"] = ["mkdir",
"-p",
self.artifacts_dir]
argo_build_util.add_task_to_dag(workflow, E2E_DAG_NAME, mkdir_step,
[checkout["name"]])
#**************************************************************************
# Run python unittests
py_tests = argo_build_util.deep_copy(task_template)
py_tests["name"] = "py-test"
py_tests["container"]["command"] = ["python",
"-m",
"kubeflow.testing.test_py_checks",
"--artifacts_dir=" + self.artifacts_dir,
# TODO(jlewi): Should we be searching
# the entire py/kubeflo/testing tree?
"--src_dir=" + self.kubeflow_testing_py
+ "kubeflow/tests"]
argo_build_util.add_task_to_dag(workflow, E2E_DAG_NAME, py_tests,
[mkdir_step["name"]])
#***************************************************************************
# py lint
#***************************************************************************
py_lint = argo_build_util.deep_copy(task_template)
py_lint["name"] = "py-lint"
py_lint["container"]["command"] = ["pytest",
"test_py_lint.py",
# I think -s mean stdout/stderr will
# print out to aid in debugging.
# Failures still appear to be captured
# and stored in the junit file.
"-s",
"--src_dir=" + self.kubeflow_testing_py,
"--rcfile=" + os.path.join(
self.testing_src_dir, ".pylintrc"),
# Test timeout in seconds.
"--timeout=500",
"--junitxml=" + self.artifacts_dir +
"/junit_py-lint.xml"]
py_lint_step = argo_build_util.add_task_to_dag(workflow, E2E_DAG_NAME,
py_lint,
[mkdir_step["name"]])
py_lint_step["container"]["workingDir"] = os.path.join(
self.testing_src_dir, "py/kubeflow/testing")
#*****************************************************************************
# create_pr_symlink
#****************************************************************************
# TODO(jlewi): run_e2e_workflow.py should probably create the PR symlink
symlink = argo_build_util.deep_copy(task_template)
symlink["name"] = "create-pr-symlink"
symlink["container"]["command"] = ["python",
"-m",
"kubeflow.testing.prow_artifacts",
"--artifacts_dir=" + self.output_dir,
"create_pr_symlink",
]
if self.bucket:
symlink["container"]["command"].append("--bucket=" + self.bucket)
argo_build_util.add_task_to_dag(workflow, E2E_DAG_NAME, symlink,
[checkout["name"]])
#*****************************************************************************
# Exit handler workflow
#*****************************************************************************
copy_artifacts = argo_build_util.deep_copy(task_template)
copy_artifacts["name"] = "copy-artifacts"
copy_artifacts["container"]["command"] = ["python",
"-m",
"kubeflow.testing.prow_artifacts",
"--artifacts_dir=" +
self.output_dir,
"copy_artifacts"]
if self.bucket:
copy_artifacts["container"]["command"].append("--bucket=" + self.bucket)
argo_build_util.add_task_to_dag(workflow, EXIT_DAG_NAME, copy_artifacts, [])
# Set the labels on all templates
workflow = argo_build_util.set_task_template_labels(workflow)
return workflow
def build(self):
self.workflow = self._build_workflow()
task_template = self._build_task_template()
py3_template = argo_build_util.deep_copy(task_template)
py3_template["container"][
"image"] = "527798164940.dkr.ecr.us-west-2.amazonaws.com/aws-kubeflow-ci/test-worker:v1.2-branch"
default_namespace = "kubeflow"
#**************************************************************************
# Checkout
# create the checkout step
checkout = argo_build_util.deep_copy(task_template)
# Construct the list of repos to checkout
list_of_repos = DEFAULT_REPOS
list_of_repos.append(self.main_repo)
list_of_repos.extend(self.extra_repos)
repos = util.combine_repos(list_of_repos)
repos_str = ','.join(
['%s@%s' % (key, value) for (key, value) in repos.items()])
# If we are using a specific branch (e.g. periodic tests for release branch)
# then we need to use depth = all; otherwise checkout out the branch
# will fail. Otherwise we checkout with depth=30. We want more than
# depth=1 because the depth will determine our ability to find the common
# ancestor which affects our ability to determine which files have changed
depth = 30
if os.getenv("BRANCH_NAME"):
logging.info("BRANCH_NAME=%s; setting detph=all",
os.getenv("BRANCH_NAME"))
depth = "all"
checkout["name"] = "checkout"
checkout["container"]["command"] = [
"/usr/local/bin/checkout_repos.sh", "--repos=" + repos_str,
"--depth={0}".format(depth), "--src_dir=" + self.src_root_dir
]
argo_build_util.add_task_to_dag(self.workflow, E2E_DAG_NAME, checkout,
[])
# Change the working directory for all subsequent steps
task_template["container"]["workingDir"] = os.path.join(
self.kfctl_pytest_dir)
py3_template["container"]["workingDir"] = os.path.join(
self.kfctl_pytest_dir)
#***************************************************************************
# create_pr_symlink
#***************************************************************************
# TODO(jlewi): run_e2e_workflow.py should probably create the PR symlink
step_name = "create-pr-symlink"
command = [
"python", "-m",
"kubeflow.testing.cloudprovider.aws.prow_artifacts",
"--artifacts_dir=" + self.output_dir, "create_pr_symlink_s3",
"--bucket=" + self.bucket
]
dependences = [checkout["name"]]
symlink = self._build_step(step_name, self.workflow, E2E_DAG_NAME,
task_template, command, dependences)
#**************************************************************************
# Run build_kfctl
step_name = "kfctl-build-deploy"
command = [
"pytest",
"kfctl_go_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
"--config_path=" + self.config_path,
"--values=" + self.values_str,
# Increase the log level so that info level log statements show up.
# TODO(https://github.com/kubeflow/testing/issues/372): If we
# set a unique artifacts dir for each workflow with the proper
# prefix that should work.
"--log-cli-level=info",
"--junitxml=" + self.artifacts_dir + "/junit_kfctl-build-test" +
self.config_name + ".xml",
# TODO(jlewi) Test suite name needs to be unique based on parameters.
"-o",
"junit_suite_name=test_kfctl_go_deploy_" + self.config_name,
"--kfctl_repo_path=" + self.src_dir,
]
dependences = [checkout["name"]]
build_kfctl = self._build_step(step_name, self.workflow, E2E_DAG_NAME,
py3_template, command, dependences)
#**************************************************************************
# Create EKS cluster for E2E test
step_name = "kfctl-create-cluster"
command = [
"pytest",
"kfctl_create_cluster_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
"--cluster_name=" + self.cluster_name,
"--eks_cluster_version=" + str(self.eks_cluster_version),
# Embedded Script in the ECR Image
"--cluster_creation_script=" +
"/usr/local/bin/create-eks-cluster.sh",
"--values=" + self.values_str,
# Increase the log level so that info level log statements show up.
# TODO(https://github.com/kubeflow/testing/issues/372): If we
# set a unique artifacts dir for each workflow with the proper
# prefix that should work.
"--log-cli-level=info",
"--junitxml=" + self.artifacts_dir + "/junit_kfctl-build-test" +
self.config_name + ".xml",
# TODO(jlewi) Test suite name needs to be unique based on parameters.
"-o",
"junit_suite_name=test_kfctl_go_deploy_" + self.config_name,
]
dependences = [checkout["name"]]
create_cluster = self._build_step(step_name, self.workflow,
E2E_DAG_NAME, py3_template, command,
dependences)
#**************************************************************************
# Deploy Kubeflow
step_name = "kfctl-deploy-kubeflow"
command = [
"pytest",
"kfctl_deploy_kubeflow_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
"--cluster_name=" + self.cluster_name,
# Embedded Script in the ECR Image
"--cluster_creation_script=" +
"/usr/local/bin/create-eks-cluster.sh",
"--config_path=" + self.config_path,
"--values=" + self.values_str,
"--build_and_apply=" + str(self.build_and_apply),
# Increase the log level so that info level log statements show up.
# TODO(https://github.com/kubeflow/testing/issues/372): If we
# set a unique artifacts dir for each workflow with the proper
# prefix that should work.
"--log-cli-level=info",
"--junitxml=" + self.artifacts_dir + "/junit_kfctl-build-test" +
self.config_name + ".xml",
# TODO(jlewi) Test suite name needs to be unique based on parameters.
"-o",
"junit_suite_name=test_kfctl_go_deploy_" + self.config_name,
"--app_path=" + self.app_dir,
"--kfctl_repo_path=" + self.src_dir,
]
dependences = [
build_kfctl["name"], create_cluster["name"], symlink["name"]
]
deploy_kf = self._build_step(step_name, self.workflow, E2E_DAG_NAME,
py3_template, command, dependences)
#**************************************************************************
# Wait for Kubeflow to be ready
step_name = "kubeflow-is-ready"
command = [
"pytest",
"kf_is_ready_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
# TODO(jlewi): We should update kf_is_ready_test to take the config
# path and then based on the KfDef spec kf_is_ready_test should
# figure out what to do.
"--use_basic_auth={0}".format(self.use_basic_auth),
# Increase the log level so that info level log statements show up.
"--log-cli-level=info",
"--junitxml=" + os.path.join(
self.artifacts_dir,
"junit_kfctl-is-ready-test-" + self.config_name + ".xml"),
# Test suite name needs to be unique based on parameters
"-o",
"junit_suite_name=test_kf_is_ready_" + self.config_name,
"--app_path=" + self.app_dir,
"--cluster_name=" + self.cluster_name,
"--namespace=" + default_namespace,
]
dependences = [deploy_kf["name"]]
kf_is_ready = self._build_step(step_name, self.workflow, E2E_DAG_NAME,
task_template, command, dependences)
#**************************************************************************
# Run functional tests
dependences = [kf_is_ready["name"]]
dependences = self._build_tests_dag(dependences=dependences)
#***********************************************************************
# Delete Kubeflow
# Putting Delete Kubeflow here is deletion functionality should be tested out of exit DAG
step_name = "kfctl-delete-wrong-host"
command = [
"pytest",
"kfctl_delete_wrong_cluster.py",
"-s",
"--log-cli-level=info",
"--timeout=1000",
"--junitxml=" + self.artifacts_dir +
"/junit_kfctl-go-delete-wrong-cluster-test.xml",
"--app_path=" + self.app_dir,
"--kfctl_path=" + self.kfctl_path,
"--cluster_name=" + self.cluster_name,
]
kfctl_delete_wrong_cluster = self._build_step(step_name, self.workflow,
E2E_DAG_NAME,
task_template, command,
dependences)
kfctl_delete_wrong_cluster["container"][
"workingDir"] = self.kfctl_pytest_dir
step_name = "kfctl-delete"
command = [
"pytest",
"kfctl_delete_test.py",
"-s",
"--log-cli-level=info",
"--timeout=1000",
"--junitxml=" + self.artifacts_dir +
"/junit_kfctl-go-delete-test.xml",
"--app_path=" + self.app_dir,
"--kfctl_path=" + self.kfctl_path,
"--cluster_name=" + self.cluster_name,
]
kfctl_delete = self._build_step(step_name, self.workflow, E2E_DAG_NAME,
task_template, command,
["kfctl-delete-wrong-host"])
kfctl_delete["container"]["workingDir"] = self.kfctl_pytest_dir
#***************************************************************************
# Exit DAG
#***************************************************************************
self._build_exit_dag()
# Set the labels on all templates
self.workflow = argo_build_util.set_task_template_labels(self.workflow)
return self.workflow
def build(self):
self.workflow = self._build_workflow()
task_template = self._build_task_template()
# **************************************************************************
# Checkout
# create the checkout step
main_repo = argo_build_util.get_repo_from_prow_env()
if not main_repo:
logging.info("Prow environment variables for repo not set")
main_repo = MAIN_REPO + "@HEAD"
logging.info("Main repository: %s", main_repo)
repos = [main_repo]
repos.extend(EXTRA_REPOS)
#***************************************************************************
# Checkout the code
checkout = argo_build_util.deep_copy(task_template)
checkout["name"] = "checkout"
checkout["container"]["command"] = [
"/usr/local/bin/checkout_repos.sh", "--repos=" + ",".join(repos),
"--src_dir=" + self.src_root_dir
]
argo_build_util.add_task_to_dag(self.workflow, E2E_DAG_NAME, checkout,
[])
#***************************************************************************
# Get credentials for the latest auto-deployed cluster
credentials = argo_build_util.deep_copy(task_template)
credentials["name"] = "get-credentials"
credentials["container"]["command"] = [
"python3",
"-m",
"kubeflow.testing."
"get_kf_testing_cluster",
"get-credentials",
]
dependencies = [checkout["name"]]
argo_build_util.add_task_to_dag(self.workflow, E2E_DAG_NAME,
credentials, dependencies)
#**************************************************************************
# Run a dag of tests
self._build_tests_dag()
# Add a task to run the dag
dependencies = [credentials["name"]]
argo_build_util.add_task_only_to_dag(self.workflow, E2E_DAG_NAME,
TESTS_DAG_NAME, TESTS_DAG_NAME,
dependencies)
# **************************************************************************
# create_pr_symlink
# ***************************************************************************
# TODO(jlewi): run_e2e_workflow.py should probably create the PR symlink
step_name = "create-pr-symlink"
command = [
"python", "-m", "kubeflow.testing.prow_artifacts",
"--artifacts_dir=" + self.output_dir, "create_pr_symlink"
]
if self.bucket:
command.append(self.bucket)
dependencies = [checkout["name"]]
self._build_step(step_name, self.workflow, E2E_DAG_NAME, task_template,
command, dependencies)
self._build_exit_dag()
# Set the labels on all templates
self.workflow = argo_build_util.set_task_template_labels(self.workflow)
return self.workflow
def create_workflow(name, namespace, **kwargs):
"""Main function which returns Argo Workflow.
:param name: Argo Workflow name.
:param namespace: Argo Workflow namespace.
:param kwargs: Argo Workflow additional arguments.
:return: Created Argo Workflow.
"""
test_dir = MOUNT_PATH + "/" + name
ecr_registry = kwargs["registry"]
builder = WorkflowBuilder(name, namespace, test_dir, ecr_registry)
# Build initial structure for the Workflow.
workflow = builder.create_init_workflow()
# Delete AWS Cluster in the exit handler step.
delete_cluster = builder.create_task_template(
task_name="delete-cluster",
exec_image=IMAGE_WORKER,
command=[
"/usr/local/bin/delete-eks-cluster.sh",
])
argo_build_util.add_task_to_dag(workflow, EXIT_HANDLER, delete_cluster, [])
# Step 1. Checkout GitHub repositories.
checkout = builder.create_task_template(
task_name="checkout",
exec_image=IMAGE_WORKER,
command=["/usr/local/bin/checkout.sh", test_dir + "/src/github.com"])
argo_build_util.add_task_to_dag(workflow, ENTRYPOINT, checkout, [])
# Step 2.1 Build all Katib images.
depends = []
for image, dockerfile in KATIB_IMAGES.items():
build_image = builder.create_task_template(
task_name="build-" + image,
exec_image=IMAGE_KANIKO,
command=[
"/kaniko/executor",
"--dockerfile={}/{}".format(builder.katib_dir, dockerfile),
"--context=dir://" + builder.katib_dir,
"--destination={}/katib/v1beta1/{}:$(PULL_PULL_SHA)".format(
ecr_registry, image)
])
argo_build_util.add_task_to_dag(workflow, ENTRYPOINT, build_image,
[checkout["name"]])
depends.append(build_image["name"])
# Step 2.2 Create AWS cluster.
create_cluster = builder.create_task_template(
task_name="create-cluster",
exec_image=IMAGE_WORKER,
command=[
"/usr/local/bin/create-eks-cluster.sh",
])
argo_build_util.add_task_to_dag(workflow, ENTRYPOINT, create_cluster,
[checkout["name"]])
depends.append(create_cluster["name"])
# Step 3. Setup Katib on AWS cluster.
setup_katib = builder.create_task_template(
task_name="setup-katib",
exec_image=IMAGE_WORKER,
command=["test/e2e/v1beta1/scripts/setup-katib.sh"])
# Installing Katib after cluster is created and images are built.
argo_build_util.add_task_to_dag(workflow, ENTRYPOINT, setup_katib, depends)
# Step 4. Run Katib Experiments.
depends = [setup_katib["name"]]
tmp_depends = []
for index, (experiment, location) in enumerate(KATIB_EXPERIMENTS.items()):
run_experiment = builder.create_task_template(
task_name="run-e2e-experiment-" + experiment,
exec_image=IMAGE_WORKER,
command=[
"test/e2e/v1beta1/scripts/run-e2e-experiment.sh", location
])
argo_build_util.add_task_to_dag(workflow, ENTRYPOINT, run_experiment,
depends)
tmp_depends.append(run_experiment["name"])
# We run only X number of Experiments at the same time. index starts with 0
if (index + 1) % PARALLEL_EXECUTION == 0:
depends, tmp_depends = tmp_depends, []
return workflow
