def build(self):
self.workflow = super(Builder, self).build()
task_template = self._build_task_template()
# Change the workflow_template labels
self.workflow["metadata"]["labels"][
"workflow_template"] = TEMPLATE_LABEL
# Add the dag to upgrade Kubeflow to the workflow
self._build_upgrade_dag()
# Add a task to the E2E dag to run the dag to upgrade Kubeflow.
dependencies = [self._run_tests_step_name]
if self._test_endpoint_step_name:
dependencies.append(self._test_endpoint_step_name)
step_name = UPGRADE_DAG_NAME
template_name = UPGRADE_DAG_NAME
argo_build_util.add_task_only_to_dag(self.workflow,
kfctl_e2e_workflow.E2E_DAG_NAME,
step_name, template_name,
dependencies)
#****************************************************************************
# Add tests DAG
#****************************************************************************
# After running upgrade we want to rerun the DAG(s) that validate the deployment is healthy
step_name = "test-after-upgrade"
template_name = kfctl_e2e_workflow.TESTS_DAG_NAME
dependencies = [UPGRADE_DAG_NAME]
argo_build_util.add_task_only_to_dag(self.workflow,
kfctl_e2e_workflow.E2E_DAG_NAME,
step_name, template_name,
dependencies)
# Test the endpoint after upgrade
if self.test_endpoint:
dependencies = [UPGRADE_DAG_NAME]
step_name = "upgraded-endpoint-ready"
argo_build_util.add_task_only_to_dag(
self.workflow, kfctl_e2e_workflow.E2E_DAG_NAME, step_name,
self._test_endpoint_template_name, dependencies)
# Reset the labels on all templates to pick up the updated workflow template label
self.workflow = argo_build_util.set_task_template_labels(self.workflow)
return self.workflow
def build(self):
self.workflow = self._build_workflow()
task_template = self._build_task_template()
#**************************************************************************
# Checkout
# create the checkout step
main_repo = argo_build_util.get_repo_from_prow_env()
if not main_repo:
logging.info("Prow environment variables for repo not set")
main_repo = MAIN_REPO + "@HEAD"
logging.info("Main repository: %s", main_repo)
repos = [main_repo]
repos.extend(EXTRA_REPOS)
checkout = argo_build_util.deep_copy(task_template)
checkout["name"] = "checkout"
checkout["container"]["command"] = [
"/usr/local/bin/checkout_repos.sh", "--repos=" + ",".join(repos),
"--src_dir=" + self.src_root_dir
]
argo_build_util.add_task_to_dag(self.workflow, E2E_DAG_NAME, checkout,
[])
# Change the workfing directory for all subsequent steps
task_template["container"]["workingDir"] = os.path.join(
self.kfctl_pytest_dir)
#**************************************************************************
# Run build_kfctl and deploy kubeflow
step_name = "kfctl-build-deploy"
command = [
"pytest",
"kfctl_go_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
"--config_path=" + self.config_path,
"--build_and_apply=" + str(self.build_and_apply),
# Increase the log level so that info level log statements show up.
# TODO(https://github.com/kubeflow/testing/issues/372): If we
# set a unique artifacts dir for each workflow with the proper
# prefix that should work.
"--log-cli-level=info",
"--junitxml=" + self.artifacts_dir + "/junit_kfctl-build-test" +
self.config_name + ".xml",
# TODO(jlewi) Test suite name needs to be unique based on parameters.
#
"-o",
"junit_suite_name=test_kfctl_go_deploy_" + self.config_name,
"--app_path=" + self.app_dir,
]
dependences = [checkout["name"]]
build_kfctl = self._build_step(step_name, self.workflow, E2E_DAG_NAME,
task_template, command, dependences)
#**************************************************************************
# Wait for Kubeflow to be ready
step_name = "kubeflow-is-ready"
command = [
"pytest",
"kf_is_ready_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
# TODO(jlewi): We should update kf_is_ready_test to take the config
# path and then based on the KfDef spec kf_is_ready_test should
# figure out what to do.
"--use_basic_auth={0}".format(self.use_basic_auth),
# TODO(jlewi): We should be using ISTIO always so can we stop
# setting this
"--use_istio=true",
# Increase the log level so that info level log statements show up.
"--log-cli-level=info",
"--junitxml=" + os.path.join(
self.artifacts_dir,
"junit_kfctl-is-ready-test-" + self.config_name + ".xml"),
# Test suite name needs to be unique based on parameters
"-o",
"junit_suite_name=test_kf_is_ready_" + self.config_name,
"--app_path=" + self.app_dir,
]
dependences = [build_kfctl["name"]]
kf_is_ready = self._build_step(step_name, self.workflow, E2E_DAG_NAME,
task_template, command, dependences)
#**************************************************************************
# Wait for endpoint to be ready
if self.test_endpoint:
step_name = "endpoint-is-ready"
command = [
"pytest",
"endpoint_ready_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
# Increase the log level so that info level log statements show up.
"--log-cli-level=info",
# Test timeout in seconds.
"--timeout=1800",
"--junitxml=" + self.artifacts_dir +
"/junit_endpoint-is-ready-test-" + self.config_name + ".xml",
# Test suite name needs to be unique based on parameters
"-o",
"junit_suite_name=test_endpoint_is_ready_" + self.config_name,
"--app_path=" + self.app_dir,
"--app_name=" + self.app_name,
]
dependencies = [build_kfctl["name"]]
endpoint_ready = self._build_step(step_name, self.workflow,
E2E_DAG_NAME, task_template,
command, dependencies)
self._build_tests_dag()
# Add a task to run the dag
dependencies = [kf_is_ready["name"]]
argo_build_util.add_task_only_to_dag(self.workflow, E2E_DAG_NAME,
TESTS_DAG_NAME, TESTS_DAG_NAME,
dependencies)
#***************************************************************************
# create_pr_symlink
#***************************************************************************
# TODO(jlewi): run_e2e_workflow.py should probably create the PR symlink
step_name = "create-pr-symlink"
command = [
"python",
"-m",
"kubeflow.testing.prow_artifacts",
"--artifacts_dir=" + self.output_dir,
"create_pr_symlink",
"--bucket=" + self.bucket,
]
dependences = [checkout["name"]]
symlink = self._build_step(step_name, self.workflow, E2E_DAG_NAME,
task_template, command, dependences)
self._build_exit_dag()
# Set the labels on all templates
self.workflow = argo_build_util.set_task_template_labels(self.workflow)
return self.workflow
def build(self):
self.workflow = self._build_workflow()
task_template = self._build_task_template()
# **************************************************************************
# Checkout
# create the checkout step
main_repo = argo_build_util.get_repo_from_prow_env()
if not main_repo:
logging.info("Prow environment variables for repo not set")
main_repo = MAIN_REPO + "@HEAD"
logging.info("Main repository: %s", main_repo)
repos = [main_repo]
repos.extend(EXTRA_REPOS)
#***************************************************************************
# Checkout the code
checkout = argo_build_util.deep_copy(task_template)
checkout["name"] = "checkout"
checkout["container"]["command"] = [
"/usr/local/bin/checkout_repos.sh", "--repos=" + ",".join(repos),
"--src_dir=" + self.src_root_dir
]
argo_build_util.add_task_to_dag(self.workflow, E2E_DAG_NAME, checkout,
[])
#***************************************************************************
# Get credentials for the latest auto-deployed cluster
credentials = argo_build_util.deep_copy(task_template)
credentials["name"] = "get-credentials"
credentials["container"]["command"] = [
"python3",
"-m",
"kubeflow.testing."
"get_kf_testing_cluster",
"get-credentials",
]
dependencies = [checkout["name"]]
argo_build_util.add_task_to_dag(self.workflow, E2E_DAG_NAME,
credentials, dependencies)
#**************************************************************************
# Run a dag of tests
self._build_tests_dag()
# Add a task to run the dag
dependencies = [credentials["name"]]
argo_build_util.add_task_only_to_dag(self.workflow, E2E_DAG_NAME,
TESTS_DAG_NAME, TESTS_DAG_NAME,
dependencies)
# **************************************************************************
# create_pr_symlink
# ***************************************************************************
# TODO(jlewi): run_e2e_workflow.py should probably create the PR symlink
step_name = "create-pr-symlink"
command = [
"python", "-m", "kubeflow.testing.prow_artifacts",
"--artifacts_dir=" + self.output_dir, "create_pr_symlink"
]
if self.bucket:
command.append(self.bucket)
dependencies = [checkout["name"]]
self._build_step(step_name, self.workflow, E2E_DAG_NAME, task_template,
command, dependencies)
self._build_exit_dag()
# Set the labels on all templates
self.workflow = argo_build_util.set_task_template_labels(self.workflow)
return self.workflow
def build(self):
self.workflow = self._build_workflow()
task_template = self._build_task_template()
py3_template = argo_build_util.deep_copy(task_template)
py3_template["container"]["image"] = "gcr.io/kubeflow-ci/test-worker-py3:e9afed1-dirty"
#**************************************************************************
# Checkout
# create the checkout step
checkout = argo_build_util.deep_copy(task_template)
# Construct the list of repos to checkout
list_of_repos = DEFAULT_REPOS
list_of_repos.append(self.main_repo)
list_of_repos.extend(self.extra_repos)
repos = util.combine_repos(list_of_repos)
repos_str = ','.join(['%s@%s' % (key, value) for (key, value) in repos.items()])
# If we are using a specific branch (e.g. periodic tests for release branch)
# then we need to use depth = all; otherwise checkout out the branch
# will fail. Otherwise we checkout with depth=30. We want more than
# depth=1 because the depth will determine our ability to find the common
# ancestor which affects our ability to determine which files have changed
depth = 30
if os.getenv("BRANCH_NAME"):
logging.info("BRANCH_NAME=%s; setting detph=all",
os.getenv("BRANCH_NAME"))
depth = "all"
checkout["name"] = "checkout"
checkout["container"]["command"] = ["/usr/local/bin/checkout_repos.sh",
"--repos=" + repos_str,
"--depth={0}".format(depth),
"--src_dir=" + self.src_root_dir]
argo_build_util.add_task_to_dag(self.workflow, E2E_DAG_NAME, checkout, [])
# Change the workfing directory for all subsequent steps
task_template["container"]["workingDir"] = os.path.join(
self.kfctl_pytest_dir)
py3_template["container"]["workingDir"] = os.path.join(self.kfctl_pytest_dir)
#**************************************************************************
# Run build_kfctl and deploy kubeflow
step_name = "kfctl-build-deploy"
command = [
"pytest",
"kfctl_go_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
"--app_name=" + self.app_name,
"--config_path=" + self.config_path,
"--values=" + self.values_str,
"--build_and_apply=" + str(self.build_and_apply),
# Increase the log level so that info level log statements show up.
# TODO(https://github.com/kubeflow/testing/issues/372): If we
# set a unique artifacts dir for each workflow with the proper
# prefix that should work.
"--log-cli-level=info",
"--junitxml=" + self.artifacts_dir + "/junit_kfctl-build-test"
+ self.config_name + ".xml",
# TODO(jlewi) Test suite name needs to be unique based on parameters.
#
"-o", "junit_suite_name=test_kfctl_go_deploy_" + self.config_name,
"--app_path=" + self.app_dir,
"--kfctl_repo_path=" + self.src_dir,
"--self_signed_cert=True",
]
dependences = [checkout["name"]]
build_kfctl = self._build_step(step_name, self.workflow, E2E_DAG_NAME,
py3_template, command, dependences)
#**************************************************************************
# Wait for Kubeflow to be ready
step_name = "kubeflow-is-ready"
command = [
"pytest",
"kf_is_ready_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
# TODO(jlewi): We should update kf_is_ready_test to take the config
# path and then based on the KfDef spec kf_is_ready_test should
# figure out what to do.
"--use_basic_auth={0}".format(self.use_basic_auth),
# TODO(jlewi): We should be using ISTIO always so can we stop
# setting this
"--use_istio=true",
# Increase the log level so that info level log statements show up.
"--log-cli-level=info",
"--junitxml=" + os.path.join(self.artifacts_dir,
"junit_kfctl-is-ready-test-" +
self.config_name + ".xml"),
# Test suite name needs to be unique based on parameters
"-o", "junit_suite_name=test_kf_is_ready_" + self.config_name,
"--app_path=" + self.app_dir,
]
dependences = [build_kfctl["name"]]
kf_is_ready = self._build_step(step_name, self.workflow, E2E_DAG_NAME, task_template,
command, dependences)
#**************************************************************************
# Wait for endpoint to be ready
if self.test_endpoint:
self._test_endpoint_step_name = "endpoint-is-ready"
command = ["pytest",
"endpoint_ready_test.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
# Increase the log level so that info level log statements show up.
"--log-cli-level=info",
"--junitxml=" + self.artifacts_dir + "/junit_endpoint-is-ready-test-" + self.config_name + ".xml",
# Test suite name needs to be unique based on parameters
"-o", "junit_suite_name=test_endpoint_is_ready_" + self.config_name,
"--app_path=" + self.app_dir,
"--app_name=" + self.app_name,
"--use_basic_auth={0}".format(self.use_basic_auth),
]
dependencies = [build_kfctl["name"]]
endpoint_ready = self._build_step(self._test_endpoint_step_name,
self.workflow, E2E_DAG_NAME, py3_template,
command, dependencies)
self._test_endpoint_template_name = endpoint_ready["name"]
#**************************************************************************
# Do kfctl apply again. This test will be skip if it's presubmit.
step_name = "kfctl-second-apply"
command = [
"pytest",
"kfctl_second_apply.py",
# I think -s mean stdout/stderr will print out to aid in debugging.
# Failures still appear to be captured and stored in the junit file.
"-s",
"--log-cli-level=info",
"--junitxml=" + os.path.join(self.artifacts_dir,
"junit_kfctl-second-apply-test-" +
self.config_name + ".xml"),
# Test suite name needs to be unique based on parameters
"-o", "junit_suite_name=test_kfctl_second_apply_" + self.config_name,
"--app_path=" + self.app_dir,
"--kfctl_path=" + self.kfctl_path,
]
if self.test_endpoint:
dependences = [kf_is_ready["name"], endpoint_ready["name"]]
else:
dependences = [kf_is_ready["name"]]
kf_second_apply = self._build_step(step_name, self.workflow, E2E_DAG_NAME, task_template,
command, dependences)
self._build_tests_dag()
# Add a task to run the dag
dependencies = [kf_is_ready["name"]]
self._run_tests_step_name = TESTS_DAG_NAME
run_tests_template_name = TESTS_DAG_NAME
argo_build_util.add_task_only_to_dag(self.workflow, E2E_DAG_NAME, self._run_tests_step_name,
run_tests_template_name,
dependencies)
#***************************************************************************
# create_pr_symlink
#***************************************************************************
# TODO(jlewi): run_e2e_workflow.py should probably create the PR symlink
step_name = "create-pr-symlink"
command = ["python",
"-m",
"kubeflow.testing.prow_artifacts",
"--artifacts_dir=" + self.output_dir,
"create_pr_symlink"]
if self.bucket:
command.append(self.bucket)
dependences = [checkout["name"]]
symlink = self._build_step(step_name, self.workflow, E2E_DAG_NAME, task_template,
command, dependences)
self._build_exit_dag()
# Set the labels on all templates
self.workflow = argo_build_util.set_task_template_labels(self.workflow)
return self.workflow
def build(self):
self.workflow = super(Builder, self).build()
task_template = self._build_task_template()
# Change the workflow_template labels
self.workflow["metadata"]["labels"][
"workflow_template"] = TEMPLATE_LABEL
# Add the dag to upgrade Kubeflow to the workflow
self._build_upgrade_dag()
# Add a task to the E2E dag to run the dag to upgrade Kubeflow.
dependencies = [self._run_tests_step_name]
if self._test_endpoint_step_name:
dependencies.append(self._test_endpoint_step_name)
step_name = UPGRADE_DAG_NAME
template_name = UPGRADE_DAG_NAME
argo_build_util.add_task_only_to_dag(self.workflow,
kfctl_e2e_workflow.E2E_DAG_NAME,
step_name, template_name,
dependencies)
# Wait for Kubeflow to be ready after upgrading
step_name = READY_AFTER_UPGRADE
template_name = "kubeflow-is-ready"
command = [
"pytest",
"kf_is_ready_test.py",
"-s",
"--log-cli-level=info",
"--junitxml=" + os.path.join(
self.artifacts_dir,
"junit_ready-after-upgrade-test-" + self.config_name + ".xml"),
"-o",
"junit_suite_name=test_ready_after_upgrade_" + self.config_name,
"--app_path=" + self.app_dir,
]
dependencies = [UPGRADE_DAG_NAME]
argo_build_util.add_task_only_to_dag(self.workflow,
kfctl_e2e_workflow.E2E_DAG_NAME,
step_name, template_name,
dependencies)
#****************************************************************************
# Add tests DAG
#****************************************************************************
# After running upgrade we want to rerun the DAG(s) that validate the deployment is healthy
step_name = "test-after-upgrade"
template_name = kfctl_e2e_workflow.TESTS_DAG_NAME
dependencies = [READY_AFTER_UPGRADE]
argo_build_util.add_task_only_to_dag(self.workflow,
kfctl_e2e_workflow.E2E_DAG_NAME,
step_name, template_name,
dependencies)
# Test the endpoint after upgrade
if self.test_endpoint:
dependencies = [UPGRADE_DAG_NAME]
step_name = "upgraded-endpoint-ready"
argo_build_util.add_task_only_to_dag(
self.workflow, kfctl_e2e_workflow.E2E_DAG_NAME, step_name,
self._test_endpoint_template_name, dependencies)
# Reset the labels on all templates to pick up the updated workflow template label
self.workflow = argo_build_util.set_task_template_labels(self.workflow)
return self.workflow
