def create_kaniko_task(self, task_template, dockerfile, context, destination, no_push=False): """ A task for building images inside a cluster container using Kaniko. If we are testing the workflow locally then we won't be pushing images to any registries. This will make it easier for people to try out and extend the code. """ kaniko = argo_build_util.deep_copy(task_template) # append the tag base-commit[0:7] if ":" not in destination: sha = os.getenv("PULL_BASE_SHA", "12341234kanikotest") base = os.getenv("PULL_BASE_REF", "master") destination += ":%s-%s" % (base, sha[0:8]) kaniko["name"] = "kaniko-build-push" kaniko["container"]["image"] = "gcr.io/kaniko-project/executor:v1.5.0" kaniko["container"]["command"] = ["/kaniko/executor"] kaniko["container"]["args"] = [ "--dockerfile=%s" % dockerfile, "--context=%s" % context, "--destination=%s" % destination ] # don't push the image to a registry if trying out the produced # Argo Workflow yaml locally if LOCAL_TESTING == "True" or no_push: kaniko["container"]["args"].append("--no-push") return kaniko
def _build_step(self, name, workflow, dag_name, task_template, command, dependences): """Syntactic sugar to add a step to the workflow""" step = argo_build_util.deep_copy(task_template) step["name"] = name step["container"]["command"] = command return argo_build_util.add_task_to_dag(workflow, dag_name, step, dependences)
def create_format_python_task(self, task_template, workingDir): format_task = argo_build_util.deep_copy(task_template) format_task["name"] = "check-python-formatting" format_task["container"]["image"] = "python:3.7-slim-buster" format_task["container"]["command"] = ["/bin/sh", "-c"] format_task["container"]["args"] = ["pip install flake8 && flake8 ."] format_task["container"]["workingDir"] = workingDir return format_task
def create_format_typescript_task(self, task_template, workingDir): format_task = argo_build_util.deep_copy(task_template) format_task["name"] = "check-frontend-formatting" format_task["container"]["image"] = "node:12.20.1-stretch-slim" format_task["container"]["command"] = ["npm"] format_task["container"]["args"] = ["run", "format:check"] format_task["container"]["workingDir"] = workingDir return format_task
def _kustomize_build_task(self, task_template): k_build = argo_build_util.deep_copy(task_template) k_build["name"] = "kustomize-build-test" k_build["container"]["image"] = "k8s.gcr.io/kustomize/kustomize:v4.1.2" k_build["container"]["args"] = ["build"] manifest_dir = ("%s/components/notebook-controller/config/" "overlays/kubeflow/") % self.src_dir k_build["container"]["workingDir"] = manifest_dir return k_build
def create_install_modules_task(self, task_template, workingDir): install = argo_build_util.deep_copy(task_template) install["name"] = "npm-modules-install" install["container"]["image"] = "node:12.20.1-stretch-slim" install["container"]["command"] = ["npm"] install["container"]["args"] = ["ci"] install["container"]["workingDir"] = workingDir return install
def _kustomize_build_task(self, task_template): k_build = argo_build_util.deep_copy(task_template) k_build["name"] = "kustomize-build-test" k_build["container"]["image"] = "k8s.gcr.io/kustomize/kustomize:v4.1.2" k_build["container"]["args"] = ["build"] manifest_dir = ("%s/components/crud-web-apps/jupyter/manifests/" "overlays/istio/") % self.src_dir k_build["container"]["workingDir"] = manifest_dir return k_build
def _create_ui_build_task(self, task_template): ui_build = argo_build_util.deep_copy(task_template) ui_build["name"] = "build-shared-ui-library" ui_build["container"]["image"] = "node:12.20.1-stretch-slim" ui_build["container"]["command"] = ["npm"] ui_build["container"]["args"] = ["run", "build"] ui_dir = ("%s/components/crud-web-apps/common/" "frontend/kubeflow-common-lib/") % self.src_dir ui_build["container"]["workingDir"] = ui_dir return ui_build
def _create_install_modules_task(self, task_template): install = argo_build_util.deep_copy(task_template) install["name"] = "npm-modules-install" install["container"]["image"] = "node:12.20.1-stretch-slim" install["container"]["command"] = ["npm"] install["container"]["args"] = ["ci"] ui_dir = ("%s/components/crud-web-apps/common/" "frontend/kubeflow-common-lib/") % self.src_dir install["container"]["workingDir"] = ui_dir return install
def _create_make_dir_task(self, task_template): """Create the directory to store the artifacts of each task""" # (jlewi) # pytest was failing trying to call makedirs. My suspicion is its # because the two steps ended up trying to create the directory at the # same time and classing. So we create a separate step to do it. mkdir_step = argo_build_util.deep_copy(task_template) mkdir_step["name"] = self.mkdir_task_name mkdir_step["container"]["command"] = ["mkdir", "-p", self.artifacts_dir] return mkdir_step
def _create_exit_handler(self, task_template): ui_build = argo_build_util.deep_copy(task_template) ui_build["name"] = "rm-node-modules" ui_build["container"]["image"] = "node:12.20.1-stretch-slim" ui_build["container"]["command"] = ["rm"] ui_build["container"]["args"] = ["-r", "node_modules"] ui_dir = ("%s/components/crud-web-apps/common/" "frontend/kubeflow-common-lib/") % self.src_dir ui_build["container"]["workingDir"] = ui_dir return ui_build
def _create_ui_tests_task(self, task_template): ui_tests = argo_build_util.deep_copy(task_template) img = "browserless/chrome:1.44-chrome-stable" ui_tests["name"] = "common-ui-tests" ui_tests["container"]["image"] = img ui_tests["container"]["command"] = ["npm"] ui_tests["container"]["args"] = ["run", "test-ci"] ui_dir = ("%s/components/crud-web-apps/common/" "frontend/kubeflow-common-lib/") % self.src_dir ui_tests["container"]["workingDir"] = ui_dir return ui_tests
def create_kaniko_task(self, task_template, dockerfile, context, destination, no_push=False): """ A task for building images inside a cluster container using Kaniko. If we are testing the workflow locally then we won't be pushing images to any registries. This will make it easier for people to try out and extend the code. """ kaniko = argo_build_util.deep_copy(task_template) # for short UUID generation alphabet = string.ascii_lowercase + string.digits # append the tag base-commit[0:7] if ":" not in destination: if self.release: with open( os.path.join("/src/kubeflow/kubeflow", "releasing/version/VERSION")) as f: version = f.read().strip() destination += ":%s" % version else: sha = os.getenv("PULL_BASE_SHA", "12341234kanikotest") base = os.getenv("PULL_BASE_REF", "master") destination += ":%s-%s" % (base, sha[0:8]) # add short UUID to step name to ensure it is unique random_suffix = ''.join(random.choices(alphabet, k=8)) kaniko["name"] = "kaniko-build-push-" + random_suffix kaniko["container"]["image"] = "gcr.io/kaniko-project/executor:v1.5.0" kaniko["container"]["command"] = ["/kaniko/executor"] kaniko["container"]["args"] = [ "--dockerfile=%s" % dockerfile, "--context=%s" % context, "--destination=%s" % destination ] # don't push the image to a registry if trying out the produced # Argo Workflow yaml locally if LOCAL_TESTING == "True" or no_push: kaniko["container"]["args"].append("--no-push") return kaniko
def _build_step(self, name, workflow, dag_name, task_template, command, dependencies): """Syntactic sugar to add a step to the workflow""" step = argo_build_util.deep_copy(task_template) step["name"] = name step["container"]["command"] = command argo_build_util.add_task_to_dag(workflow, dag_name, step, dependencies) # Return the newly created template; add_task_to_dag makes a copy of the template # So we need to fetch it from the workflow spec. for t in workflow["spec"]["templates"]: if t["name"] == name: return t return None
def _create_checkout_task(self, task_template): """Checkout the kubeflow/testing and kubeflow/kubeflow code""" main_repo = argo_build_util.get_repo_from_prow_env() if not main_repo: logging.info("Prow environment variables for repo not set") main_repo = "kubeflow/testing@HEAD" logging.info("Main repository: %s", main_repo) repos = [main_repo] checkout = argo_build_util.deep_copy(task_template) checkout["name"] = "checkout" checkout["container"]["command"] = [ "/usr/local/bin/checkout_repos.sh", "--repos=" + ",".join(repos), "--src_dir=" + self.src_root_dir, ] return checkout
def build(self): self.workflow = self._build_workflow() task_template = self._build_task_template() #************************************************************************** # Checkout # create the checkout step main_repo = argo_build_util.get_repo_from_prow_env() if not main_repo: logging.info("Prow environment variables for repo not set") main_repo = MAIN_REPO + "@HEAD" logging.info("Main repository: %s", main_repo) repos = [main_repo] repos.extend(EXTRA_REPOS) checkout = argo_build_util.deep_copy(task_template) checkout["name"] = "checkout" checkout["container"]["command"] = [ "/usr/local/bin/checkout_repos.sh", "--repos=" + ",".join(repos), "--src_dir=" + self.src_root_dir ] argo_build_util.add_task_to_dag(self.workflow, E2E_DAG_NAME, checkout, []) # Change the workfing directory for all subsequent steps task_template["container"]["workingDir"] = os.path.join( self.kfctl_pytest_dir) #************************************************************************** # Run build_kfctl and deploy kubeflow step_name = "kfctl-build-deploy" command = [ "pytest", "kfctl_go_test.py", # I think -s mean stdout/stderr will print out to aid in debugging. # Failures still appear to be captured and stored in the junit file. "-s", "--config_path=" + self.config_path, "--build_and_apply=" + str(self.build_and_apply), # Increase the log level so that info level log statements show up. # TODO(https://github.com/kubeflow/testing/issues/372): If we # set a unique artifacts dir for each workflow with the proper # prefix that should work. "--log-cli-level=info", "--junitxml=" + self.artifacts_dir + "/junit_kfctl-build-test" + self.config_name + ".xml", # TODO(jlewi) Test suite name needs to be unique based on parameters. # "-o", "junit_suite_name=test_kfctl_go_deploy_" + self.config_name, "--app_path=" + self.app_dir, ] dependences = [checkout["name"]] build_kfctl = self._build_step(step_name, self.workflow, E2E_DAG_NAME, task_template, command, dependences) #************************************************************************** # Wait for Kubeflow to be ready step_name = "kubeflow-is-ready" command = [ "pytest", "kf_is_ready_test.py", # I think -s mean stdout/stderr will print out to aid in debugging. # Failures still appear to be captured and stored in the junit file. "-s", # TODO(jlewi): We should update kf_is_ready_test to take the config # path and then based on the KfDef spec kf_is_ready_test should # figure out what to do. "--use_basic_auth={0}".format(self.use_basic_auth), # TODO(jlewi): We should be using ISTIO always so can we stop # setting this "--use_istio=true", # Increase the log level so that info level log statements show up. "--log-cli-level=info", "--junitxml=" + os.path.join( self.artifacts_dir, "junit_kfctl-is-ready-test-" + self.config_name + ".xml"), # Test suite name needs to be unique based on parameters "-o", "junit_suite_name=test_kf_is_ready_" + self.config_name, "--app_path=" + self.app_dir, ] dependences = [build_kfctl["name"]] kf_is_ready = self._build_step(step_name, self.workflow, E2E_DAG_NAME, task_template, command, dependences) #************************************************************************** # Wait for endpoint to be ready if self.test_endpoint: step_name = "endpoint-is-ready" command = [ "pytest", "endpoint_ready_test.py", # I think -s mean stdout/stderr will print out to aid in debugging. # Failures still appear to be captured and stored in the junit file. "-s", # Increase the log level so that info level log statements show up. "--log-cli-level=info", # Test timeout in seconds. "--timeout=1800", "--junitxml=" + self.artifacts_dir + "/junit_endpoint-is-ready-test-" + self.config_name + ".xml", # Test suite name needs to be unique based on parameters "-o", "junit_suite_name=test_endpoint_is_ready_" + self.config_name, "--app_path=" + self.app_dir, "--app_name=" + self.app_name, ] dependencies = [build_kfctl["name"]] endpoint_ready = self._build_step(step_name, self.workflow, E2E_DAG_NAME, task_template, command, dependencies) self._build_tests_dag() # Add a task to run the dag dependencies = [kf_is_ready["name"]] argo_build_util.add_task_only_to_dag(self.workflow, E2E_DAG_NAME, TESTS_DAG_NAME, TESTS_DAG_NAME, dependencies) #*************************************************************************** # create_pr_symlink #*************************************************************************** # TODO(jlewi): run_e2e_workflow.py should probably create the PR symlink step_name = "create-pr-symlink" command = [ "python", "-m", "kubeflow.testing.prow_artifacts", "--artifacts_dir=" + self.output_dir, "create_pr_symlink", "--bucket=" + self.bucket, ] dependences = [checkout["name"]] symlink = self._build_step(step_name, self.workflow, E2E_DAG_NAME, task_template, command, dependences) self._build_exit_dag() # Set the labels on all templates self.workflow = argo_build_util.set_task_template_labels(self.workflow) return self.workflow
def build(self): self.workflow = self._build_workflow() task_template = self._build_task_template() py3_template = argo_build_util.deep_copy(task_template) py3_template["container"]["image"] = "gcr.io/kubeflow-ci/test-worker-py3:e9afed1-dirty" #************************************************************************** # Checkout # create the checkout step checkout = argo_build_util.deep_copy(task_template) # Construct the list of repos to checkout list_of_repos = DEFAULT_REPOS list_of_repos.append(self.main_repo) list_of_repos.extend(self.extra_repos) repos = util.combine_repos(list_of_repos) repos_str = ','.join(['%s@%s' % (key, value) for (key, value) in repos.items()]) # If we are using a specific branch (e.g. periodic tests for release branch) # then we need to use depth = all; otherwise checkout out the branch # will fail. Otherwise we checkout with depth=30. We want more than # depth=1 because the depth will determine our ability to find the common # ancestor which affects our ability to determine which files have changed depth = 30 if os.getenv("BRANCH_NAME"): logging.info("BRANCH_NAME=%s; setting detph=all", os.getenv("BRANCH_NAME")) depth = "all" checkout["name"] = "checkout" checkout["container"]["command"] = ["/usr/local/bin/checkout_repos.sh", "--repos=" + repos_str, "--depth={0}".format(depth), "--src_dir=" + self.src_root_dir] argo_build_util.add_task_to_dag(self.workflow, E2E_DAG_NAME, checkout, []) # Change the workfing directory for all subsequent steps task_template["container"]["workingDir"] = os.path.join( self.kfctl_pytest_dir) py3_template["container"]["workingDir"] = os.path.join(self.kfctl_pytest_dir) #************************************************************************** # Run build_kfctl and deploy kubeflow step_name = "kfctl-build-deploy" command = [ "pytest", "kfctl_go_test.py", # I think -s mean stdout/stderr will print out to aid in debugging. # Failures still appear to be captured and stored in the junit file. "-s", "--app_name=" + self.app_name, "--config_path=" + self.config_path, "--values=" + self.values_str, "--build_and_apply=" + str(self.build_and_apply), # Increase the log level so that info level log statements show up. # TODO(https://github.com/kubeflow/testing/issues/372): If we # set a unique artifacts dir for each workflow with the proper # prefix that should work. "--log-cli-level=info", "--junitxml=" + self.artifacts_dir + "/junit_kfctl-build-test" + self.config_name + ".xml", # TODO(jlewi) Test suite name needs to be unique based on parameters. # "-o", "junit_suite_name=test_kfctl_go_deploy_" + self.config_name, "--app_path=" + self.app_dir, "--kfctl_repo_path=" + self.src_dir, "--self_signed_cert=True", ] dependences = [checkout["name"]] build_kfctl = self._build_step(step_name, self.workflow, E2E_DAG_NAME, py3_template, command, dependences) #************************************************************************** # Wait for Kubeflow to be ready step_name = "kubeflow-is-ready" command = [ "pytest", "kf_is_ready_test.py", # I think -s mean stdout/stderr will print out to aid in debugging. # Failures still appear to be captured and stored in the junit file. "-s", # TODO(jlewi): We should update kf_is_ready_test to take the config # path and then based on the KfDef spec kf_is_ready_test should # figure out what to do. "--use_basic_auth={0}".format(self.use_basic_auth), # TODO(jlewi): We should be using ISTIO always so can we stop # setting this "--use_istio=true", # Increase the log level so that info level log statements show up. "--log-cli-level=info", "--junitxml=" + os.path.join(self.artifacts_dir, "junit_kfctl-is-ready-test-" + self.config_name + ".xml"), # Test suite name needs to be unique based on parameters "-o", "junit_suite_name=test_kf_is_ready_" + self.config_name, "--app_path=" + self.app_dir, ] dependences = [build_kfctl["name"]] kf_is_ready = self._build_step(step_name, self.workflow, E2E_DAG_NAME, task_template, command, dependences) #************************************************************************** # Wait for endpoint to be ready if self.test_endpoint: self._test_endpoint_step_name = "endpoint-is-ready" command = ["pytest", "endpoint_ready_test.py", # I think -s mean stdout/stderr will print out to aid in debugging. # Failures still appear to be captured and stored in the junit file. "-s", # Increase the log level so that info level log statements show up. "--log-cli-level=info", "--junitxml=" + self.artifacts_dir + "/junit_endpoint-is-ready-test-" + self.config_name + ".xml", # Test suite name needs to be unique based on parameters "-o", "junit_suite_name=test_endpoint_is_ready_" + self.config_name, "--app_path=" + self.app_dir, "--app_name=" + self.app_name, "--use_basic_auth={0}".format(self.use_basic_auth), ] dependencies = [build_kfctl["name"]] endpoint_ready = self._build_step(self._test_endpoint_step_name, self.workflow, E2E_DAG_NAME, py3_template, command, dependencies) self._test_endpoint_template_name = endpoint_ready["name"] #************************************************************************** # Do kfctl apply again. This test will be skip if it's presubmit. step_name = "kfctl-second-apply" command = [ "pytest", "kfctl_second_apply.py", # I think -s mean stdout/stderr will print out to aid in debugging. # Failures still appear to be captured and stored in the junit file. "-s", "--log-cli-level=info", "--junitxml=" + os.path.join(self.artifacts_dir, "junit_kfctl-second-apply-test-" + self.config_name + ".xml"), # Test suite name needs to be unique based on parameters "-o", "junit_suite_name=test_kfctl_second_apply_" + self.config_name, "--app_path=" + self.app_dir, "--kfctl_path=" + self.kfctl_path, ] if self.test_endpoint: dependences = [kf_is_ready["name"], endpoint_ready["name"]] else: dependences = [kf_is_ready["name"]] kf_second_apply = self._build_step(step_name, self.workflow, E2E_DAG_NAME, task_template, command, dependences) self._build_tests_dag() # Add a task to run the dag dependencies = [kf_is_ready["name"]] self._run_tests_step_name = TESTS_DAG_NAME run_tests_template_name = TESTS_DAG_NAME argo_build_util.add_task_only_to_dag(self.workflow, E2E_DAG_NAME, self._run_tests_step_name, run_tests_template_name, dependencies) #*************************************************************************** # create_pr_symlink #*************************************************************************** # TODO(jlewi): run_e2e_workflow.py should probably create the PR symlink step_name = "create-pr-symlink" command = ["python", "-m", "kubeflow.testing.prow_artifacts", "--artifacts_dir=" + self.output_dir, "create_pr_symlink"] if self.bucket: command.append(self.bucket) dependences = [checkout["name"]] symlink = self._build_step(step_name, self.workflow, E2E_DAG_NAME, task_template, command, dependences) self._build_exit_dag() # Set the labels on all templates self.workflow = argo_build_util.set_task_template_labels(self.workflow) return self.workflow
def build(self): workflow = self._build_workflow() task_template = self._build_task_template() #************************************************************************** # Checkout # create the checkout step main_repo = argo_build_util.get_repo_from_prow_env() if not main_repo: logging.info("Prow environment variables for repo not set") main_repo = "kubeflow/testing@HEAD" logging.info("Main repository: %s", main_repo) repos = [main_repo] checkout = argo_build_util.deep_copy(task_template) checkout["name"] = "checkout" checkout["container"]["command"] = ["/usr/local/bin/checkout_repos.sh", "--repos=" + ",".join(repos), "--src_dir=" + self.src_root_dir] argo_build_util.add_task_to_dag(workflow, E2E_DAG_NAME, checkout, []) #************************************************************************** # Make dir # pytest was failing trying to call makedirs. My suspicion is its # because the two steps ended up trying to create the directory at the # same time and classing. So we create a separate step to do it. mkdir_step = argo_build_util.deep_copy(task_template) mkdir_step["name"] = "make-artifacts-dir" mkdir_step["container"]["command"] = ["mkdir", "-p", self.artifacts_dir] argo_build_util.add_task_to_dag(workflow, E2E_DAG_NAME, mkdir_step, [checkout["name"]]) #************************************************************************** # Run python unittests py_tests = argo_build_util.deep_copy(task_template) py_tests["name"] = "py-test" py_tests["container"]["command"] = ["python", "-m", "kubeflow.testing.test_py_checks", "--artifacts_dir=" + self.artifacts_dir, # TODO(jlewi): Should we be searching # the entire py/kubeflo/testing tree? "--src_dir=" + self.kubeflow_testing_py + "kubeflow/tests"] argo_build_util.add_task_to_dag(workflow, E2E_DAG_NAME, py_tests, [mkdir_step["name"]]) #*************************************************************************** # py lint #*************************************************************************** py_lint = argo_build_util.deep_copy(task_template) py_lint["name"] = "py-lint" py_lint["container"]["command"] = ["pytest", "test_py_lint.py", # I think -s mean stdout/stderr will # print out to aid in debugging. # Failures still appear to be captured # and stored in the junit file. "-s", "--src_dir=" + self.kubeflow_testing_py, "--rcfile=" + os.path.join( self.testing_src_dir, ".pylintrc"), # Test timeout in seconds. "--timeout=500", "--junitxml=" + self.artifacts_dir + "/junit_py-lint.xml"] py_lint_step = argo_build_util.add_task_to_dag(workflow, E2E_DAG_NAME, py_lint, [mkdir_step["name"]]) py_lint_step["container"]["workingDir"] = os.path.join( self.testing_src_dir, "py/kubeflow/testing") #***************************************************************************** # create_pr_symlink #**************************************************************************** # TODO(jlewi): run_e2e_workflow.py should probably create the PR symlink symlink = argo_build_util.deep_copy(task_template) symlink["name"] = "create-pr-symlink" symlink["container"]["command"] = ["python", "-m", "kubeflow.testing.prow_artifacts", "--artifacts_dir=" + self.output_dir, "create_pr_symlink", ] if self.bucket: symlink["container"]["command"].append("--bucket=" + self.bucket) argo_build_util.add_task_to_dag(workflow, E2E_DAG_NAME, symlink, [checkout["name"]]) #***************************************************************************** # Exit handler workflow #***************************************************************************** copy_artifacts = argo_build_util.deep_copy(task_template) copy_artifacts["name"] = "copy-artifacts" copy_artifacts["container"]["command"] = ["python", "-m", "kubeflow.testing.prow_artifacts", "--artifacts_dir=" + self.output_dir, "copy_artifacts"] if self.bucket: copy_artifacts["container"]["command"].append("--bucket=" + self.bucket) argo_build_util.add_task_to_dag(workflow, EXIT_DAG_NAME, copy_artifacts, []) # Set the labels on all templates workflow = argo_build_util.set_task_template_labels(workflow) return workflow
def build(self): self.workflow = self._build_workflow() task_template = self._build_task_template() py3_template = argo_build_util.deep_copy(task_template) py3_template["container"][ "image"] = "527798164940.dkr.ecr.us-west-2.amazonaws.com/aws-kubeflow-ci/test-worker:v1.2-branch" default_namespace = "kubeflow" #************************************************************************** # Checkout # create the checkout step checkout = argo_build_util.deep_copy(task_template) # Construct the list of repos to checkout list_of_repos = DEFAULT_REPOS list_of_repos.append(self.main_repo) list_of_repos.extend(self.extra_repos) repos = util.combine_repos(list_of_repos) repos_str = ','.join( ['%s@%s' % (key, value) for (key, value) in repos.items()]) # If we are using a specific branch (e.g. periodic tests for release branch) # then we need to use depth = all; otherwise checkout out the branch # will fail. Otherwise we checkout with depth=30. We want more than # depth=1 because the depth will determine our ability to find the common # ancestor which affects our ability to determine which files have changed depth = 30 if os.getenv("BRANCH_NAME"): logging.info("BRANCH_NAME=%s; setting detph=all", os.getenv("BRANCH_NAME")) depth = "all" checkout["name"] = "checkout" checkout["container"]["command"] = [ "/usr/local/bin/checkout_repos.sh", "--repos=" + repos_str, "--depth={0}".format(depth), "--src_dir=" + self.src_root_dir ] argo_build_util.add_task_to_dag(self.workflow, E2E_DAG_NAME, checkout, []) # Change the working directory for all subsequent steps task_template["container"]["workingDir"] = os.path.join( self.kfctl_pytest_dir) py3_template["container"]["workingDir"] = os.path.join( self.kfctl_pytest_dir) #*************************************************************************** # create_pr_symlink #*************************************************************************** # TODO(jlewi): run_e2e_workflow.py should probably create the PR symlink step_name = "create-pr-symlink" command = [ "python", "-m", "kubeflow.testing.cloudprovider.aws.prow_artifacts", "--artifacts_dir=" + self.output_dir, "create_pr_symlink_s3", "--bucket=" + self.bucket ] dependences = [checkout["name"]] symlink = self._build_step(step_name, self.workflow, E2E_DAG_NAME, task_template, command, dependences) #************************************************************************** # Run build_kfctl step_name = "kfctl-build-deploy" command = [ "pytest", "kfctl_go_test.py", # I think -s mean stdout/stderr will print out to aid in debugging. # Failures still appear to be captured and stored in the junit file. "-s", "--config_path=" + self.config_path, "--values=" + self.values_str, # Increase the log level so that info level log statements show up. # TODO(https://github.com/kubeflow/testing/issues/372): If we # set a unique artifacts dir for each workflow with the proper # prefix that should work. "--log-cli-level=info", "--junitxml=" + self.artifacts_dir + "/junit_kfctl-build-test" + self.config_name + ".xml", # TODO(jlewi) Test suite name needs to be unique based on parameters. "-o", "junit_suite_name=test_kfctl_go_deploy_" + self.config_name, "--kfctl_repo_path=" + self.src_dir, ] dependences = [checkout["name"]] build_kfctl = self._build_step(step_name, self.workflow, E2E_DAG_NAME, py3_template, command, dependences) #************************************************************************** # Create EKS cluster for E2E test step_name = "kfctl-create-cluster" command = [ "pytest", "kfctl_create_cluster_test.py", # I think -s mean stdout/stderr will print out to aid in debugging. # Failures still appear to be captured and stored in the junit file. "-s", "--cluster_name=" + self.cluster_name, "--eks_cluster_version=" + str(self.eks_cluster_version), # Embedded Script in the ECR Image "--cluster_creation_script=" + "/usr/local/bin/create-eks-cluster.sh", "--values=" + self.values_str, # Increase the log level so that info level log statements show up. # TODO(https://github.com/kubeflow/testing/issues/372): If we # set a unique artifacts dir for each workflow with the proper # prefix that should work. "--log-cli-level=info", "--junitxml=" + self.artifacts_dir + "/junit_kfctl-build-test" + self.config_name + ".xml", # TODO(jlewi) Test suite name needs to be unique based on parameters. "-o", "junit_suite_name=test_kfctl_go_deploy_" + self.config_name, ] dependences = [checkout["name"]] create_cluster = self._build_step(step_name, self.workflow, E2E_DAG_NAME, py3_template, command, dependences) #************************************************************************** # Deploy Kubeflow step_name = "kfctl-deploy-kubeflow" command = [ "pytest", "kfctl_deploy_kubeflow_test.py", # I think -s mean stdout/stderr will print out to aid in debugging. # Failures still appear to be captured and stored in the junit file. "-s", "--cluster_name=" + self.cluster_name, # Embedded Script in the ECR Image "--cluster_creation_script=" + "/usr/local/bin/create-eks-cluster.sh", "--config_path=" + self.config_path, "--values=" + self.values_str, "--build_and_apply=" + str(self.build_and_apply), # Increase the log level so that info level log statements show up. # TODO(https://github.com/kubeflow/testing/issues/372): If we # set a unique artifacts dir for each workflow with the proper # prefix that should work. "--log-cli-level=info", "--junitxml=" + self.artifacts_dir + "/junit_kfctl-build-test" + self.config_name + ".xml", # TODO(jlewi) Test suite name needs to be unique based on parameters. "-o", "junit_suite_name=test_kfctl_go_deploy_" + self.config_name, "--app_path=" + self.app_dir, "--kfctl_repo_path=" + self.src_dir, ] dependences = [ build_kfctl["name"], create_cluster["name"], symlink["name"] ] deploy_kf = self._build_step(step_name, self.workflow, E2E_DAG_NAME, py3_template, command, dependences) #************************************************************************** # Wait for Kubeflow to be ready step_name = "kubeflow-is-ready" command = [ "pytest", "kf_is_ready_test.py", # I think -s mean stdout/stderr will print out to aid in debugging. # Failures still appear to be captured and stored in the junit file. "-s", # TODO(jlewi): We should update kf_is_ready_test to take the config # path and then based on the KfDef spec kf_is_ready_test should # figure out what to do. "--use_basic_auth={0}".format(self.use_basic_auth), # Increase the log level so that info level log statements show up. "--log-cli-level=info", "--junitxml=" + os.path.join( self.artifacts_dir, "junit_kfctl-is-ready-test-" + self.config_name + ".xml"), # Test suite name needs to be unique based on parameters "-o", "junit_suite_name=test_kf_is_ready_" + self.config_name, "--app_path=" + self.app_dir, "--cluster_name=" + self.cluster_name, "--namespace=" + default_namespace, ] dependences = [deploy_kf["name"]] kf_is_ready = self._build_step(step_name, self.workflow, E2E_DAG_NAME, task_template, command, dependences) #************************************************************************** # Run functional tests dependences = [kf_is_ready["name"]] dependences = self._build_tests_dag(dependences=dependences) #*********************************************************************** # Delete Kubeflow # Putting Delete Kubeflow here is deletion functionality should be tested out of exit DAG step_name = "kfctl-delete-wrong-host" command = [ "pytest", "kfctl_delete_wrong_cluster.py", "-s", "--log-cli-level=info", "--timeout=1000", "--junitxml=" + self.artifacts_dir + "/junit_kfctl-go-delete-wrong-cluster-test.xml", "--app_path=" + self.app_dir, "--kfctl_path=" + self.kfctl_path, "--cluster_name=" + self.cluster_name, ] kfctl_delete_wrong_cluster = self._build_step(step_name, self.workflow, E2E_DAG_NAME, task_template, command, dependences) kfctl_delete_wrong_cluster["container"][ "workingDir"] = self.kfctl_pytest_dir step_name = "kfctl-delete" command = [ "pytest", "kfctl_delete_test.py", "-s", "--log-cli-level=info", "--timeout=1000", "--junitxml=" + self.artifacts_dir + "/junit_kfctl-go-delete-test.xml", "--app_path=" + self.app_dir, "--kfctl_path=" + self.kfctl_path, "--cluster_name=" + self.cluster_name, ] kfctl_delete = self._build_step(step_name, self.workflow, E2E_DAG_NAME, task_template, command, ["kfctl-delete-wrong-host"]) kfctl_delete["container"]["workingDir"] = self.kfctl_pytest_dir #*************************************************************************** # Exit DAG #*************************************************************************** self._build_exit_dag() # Set the labels on all templates self.workflow = argo_build_util.set_task_template_labels(self.workflow) return self.workflow
def build(self): self.workflow = self._build_workflow() task_template = self._build_task_template() # ************************************************************************** # Checkout # create the checkout step main_repo = argo_build_util.get_repo_from_prow_env() if not main_repo: logging.info("Prow environment variables for repo not set") main_repo = MAIN_REPO + "@HEAD" logging.info("Main repository: %s", main_repo) repos = [main_repo] repos.extend(EXTRA_REPOS) #*************************************************************************** # Checkout the code checkout = argo_build_util.deep_copy(task_template) checkout["name"] = "checkout" checkout["container"]["command"] = [ "/usr/local/bin/checkout_repos.sh", "--repos=" + ",".join(repos), "--src_dir=" + self.src_root_dir ] argo_build_util.add_task_to_dag(self.workflow, E2E_DAG_NAME, checkout, []) #*************************************************************************** # Get credentials for the latest auto-deployed cluster credentials = argo_build_util.deep_copy(task_template) credentials["name"] = "get-credentials" credentials["container"]["command"] = [ "python3", "-m", "kubeflow.testing." "get_kf_testing_cluster", "get-credentials", ] dependencies = [checkout["name"]] argo_build_util.add_task_to_dag(self.workflow, E2E_DAG_NAME, credentials, dependencies) #************************************************************************** # Run a dag of tests self._build_tests_dag() # Add a task to run the dag dependencies = [credentials["name"]] argo_build_util.add_task_only_to_dag(self.workflow, E2E_DAG_NAME, TESTS_DAG_NAME, TESTS_DAG_NAME, dependencies) # ************************************************************************** # create_pr_symlink # *************************************************************************** # TODO(jlewi): run_e2e_workflow.py should probably create the PR symlink step_name = "create-pr-symlink" command = [ "python", "-m", "kubeflow.testing.prow_artifacts", "--artifacts_dir=" + self.output_dir, "create_pr_symlink" ] if self.bucket: command.append(self.bucket) dependencies = [checkout["name"]] self._build_step(step_name, self.workflow, E2E_DAG_NAME, task_template, command, dependencies) self._build_exit_dag() # Set the labels on all templates self.workflow = argo_build_util.set_task_template_labels(self.workflow) return self.workflow