def create_task_template(self, task_name, exec_image, command):
"""Creates template for all the Workflow tasks.
:param task_name: Template name for the task.
:param exec_image: Container image to execute the task.
:param command: List of container commands.
:return: Created task template.
"""
# Container environment variables.
# TODO (andreyvelich): Add PYTHONPATH ?
env = [
{
"name": "AWS_ACCESS_KEY_ID",
"valueFrom": {
"secretKeyRef": {
"name": "aws-credentials",
"key": "AWS_ACCESS_KEY_ID"
}
}
},
{
"name": "AWS_SECRET_ACCESS_KEY",
"valueFrom": {
"secretKeyRef": {
"name": "aws-credentials",
"key": "AWS_SECRET_ACCESS_KEY"
}
}
},
{
"name": "AWS_REGION",
"value": "us-west-2"
},
{
"name": "CLUSTER_NAME",
"value": self.workflow_name
},
{
"name": "EKS_CLUSTER_VERSION",
"value": "1.19"
},
{
"name": "ECR_REGISTRY",
"value": self.ecr_registry
},
{
"name": "GIT_TOKEN",
"valueFrom": {
"secretKeyRef": {
"name": "github-token",
"key": "github_token"
}
}
},
{
"name": "MANIFESTS_DIR",
"value": self.manifest_dir
},
{
"name": "EXTRA_REPOS",
"value": "kubeflow/testing@HEAD;kubeflow/[email protected]"
},
# Set GOPATH to test_dir because Katib repo is located under /src/github.com/kubeflow/katib
{
"name": "GOPATH",
"value": self.test_dir
}
]
# Container volume mounts.
volume_mounts = [
{
"name": VOLUME_TEST_DATA,
"mountPath": MOUNT_PATH
},
{
"name": VOLUME_GITHUB_TOKEN,
"mountPath": "/secret/github-token"
},
{
"name": VOLUME_AWS_SECRET,
"mountPath": "/root/.aws/"
},
{
"name": VOLUME_DOCKER_CONFIG,
"mountPath": "/kaniko/.docker/"
},
]
task_template = {
"name": task_name,
# Each container can be alive for 40 minutes.
"retryStrategy": {
"limit": "3",
"retryPolicy": "Always",
"backoff": {
"duration": "1",
"factor": "2",
"maxDuration": "1m",
},
},
"container": {
"command": command,
"image": exec_image,
"workingDir": self.katib_dir,
"env": env,
"volumeMounts": volume_mounts,
}
}
# Add prow env to the task template.
prow_env_dict = argo_build_util.get_prow_dict()
for k, v in prow_env_dict.items():
task_template["container"]["env"].append({"name": k, "value": v})
return task_template
# This is the main dag with the entrypoint
E2E_DAG_NAME = "e2e"
EXIT_DAG_NAME = "exit-handler"
# This is a sub dag containing the suite of tests to run against
# Kubeflow deployment
TESTS_DAG_NAME = "gke-tests"
TEMPLATE_LABEL = "examples_e2e"
MAIN_REPO = "kubeflow/examples"
EXTRA_REPOS = ["kubeflow/testing@HEAD", "kubeflow/tf-operator@HEAD"]
PROW_DICT = argo_build_util.get_prow_dict()
class Builder:
def __init__(self,
name=None,
namespace=None,
test_target_name=None,
bucket=None,
**kwargs): # pylint: disable=unused-argument
"""Initialize a builder.
Args:
name: Name for the workflow.
namespace: Namespace for the workflow.
test_target_name: (Optional) Name to use as the test target to group