def setup_ks_app(test_dir, src_root_dir, namespace, github_token, api_client):
"""Create a ksonnet app for Kubeflow"""
util.makedirs(test_dir)
logging.info("Using test directory: %s", test_dir)
namespace_name = namespace
namespace = _setup_test(api_client, namespace_name)
logging.info("Using namespace: %s", namespace)
if github_token:
logging.info("Setting GITHUB_TOKEN to %s.", github_token)
# Set a GITHUB_TOKEN so that we don't rate limited by GitHub;
# see: https://github.com/ksonnet/ksonnet/issues/233
os.environ["GITHUB_TOKEN"] = github_token
if not os.getenv("GITHUB_TOKEN"):
logging.warning(
"GITHUB_TOKEN not set; you will probably hit Github API "
"limits.")
kubeflow_registry = os.path.join(src_root_dir, "kubeflow", "kubeflow",
"kubeflow")
kubebench_registry = os.path.join(src_root_dir, "kubeflow", "kubebench",
"kubebench")
app_name = "kubeflow-test-" + uuid.uuid4().hex[0:4]
app_dir = os.path.join(test_dir, app_name)
# Initialize a ksonnet app.
util.run(["ks", "init", app_name], cwd=test_dir)
# Set the default namespace.
util.run(["ks", "env", "set", "default", "--namespace=" + namespace_name],
cwd=app_dir)
# Add required registries
registries = {
"kubeflow": kubeflow_registry,
"kubebench": kubebench_registry
}
for r in registries:
util.run(["ks", "registry", "add", r, registries[r]], cwd=app_dir)
# Install required packages
packages = [
"kubeflow/core", "kubeflow/argo", "kubebench/kubebench-job",
"kubebench/nfs-server", "kubebench/nfs-volume", "kubeflow/tf-training"
]
for p in packages:
util.run(["ks", "pkg", "install", p], cwd=app_dir)
return app_dir
def setup_ks_app(base_app_dir, namespace, api_client, kubeflow_registry,
kubebench_registry):
"""Create a ksonnet app for Kubeflow"""
util.makedirs(base_app_dir)
logging.info("Using directory to initiate ksonnet application: %s",
base_app_dir)
namespace_name = namespace
namespace = _setup_test(api_client, namespace_name)
logging.info("Using namespace: %s", namespace)
if not os.getenv("GITHUB_TOKEN"):
logging.warning(
"GITHUB_TOKEN not set; you will probably hit Github API "
"limits.")
timestamp = datetime.datetime.now()
app_name = "ks-app"
app_dir = os.path.join(base_app_dir, app_name)
# Initialize a ksonnet app.
util.run(["ks", "init", app_name], cwd=base_app_dir)
# Set the default namespace.
util.run(["ks", "env", "set", "default", "--namespace=" + namespace_name],
cwd=app_dir)
# Add required registries
registries = {
"kubeflow": kubeflow_registry,
"kubebench": kubebench_registry
}
for r in registries:
util.run(["ks", "registry", "add", r, registries[r]], cwd=app_dir)
# Install required packages
packages = [
"kubeflow/common", "kubeflow/argo", "kubeflow/tf-training",
"kubeflow/kubebench", "kubeflow/mpi-job"
]
for p in packages:
util.run(["ks", "pkg", "install", p], cwd=app_dir)
return app_dir
def setup_kubeflow_ks_app(dir, namespace, github_token, api_client):
"""Create a ksonnet app for Kubeflow"""
util.makedirs(dir)
logging.info("Using test directory: %s", dir)
namespace_name = namespace
namespace = _setup_test(api_client, namespace_name)
logging.info("Using namespace: %s", namespace)
if github_token:
logging.info("Setting GITHUB_TOKEN to %s.", github_token)
# Set a GITHUB_TOKEN so that we don't rate limited by GitHub;
# see: https://github.com/ksonnet/ksonnet/issues/233
os.environ["GITHUB_TOKEN"] = github_token
if not os.getenv("GITHUB_TOKEN"):
logging.warning(
"GITHUB_TOKEN not set; you will probably hit Github API "
"limits.")
# Initialize a ksonnet app.
app_name = "kubeflow-test-" + uuid.uuid4().hex[0:4]
util.run([
"ks",
"init",
app_name,
], cwd=dir)
app_dir = os.path.join(dir, app_name)
kubeflow_registry = "github.com/kubeflow/kubeflow/tree/master/kubeflow"
util.run(["ks", "registry", "add", "kubeflow", kubeflow_registry],
cwd=app_dir)
# Install required packages
packages = [
"kubeflow/core", "kubeflow/tf-serving", "kubeflow/tf-job",
"kubeflow/pytorch-job", "kubeflow/argo"
]
# Instead of installing packages we edit the app.yaml file directly
#for p in packages:
# util.run(["ks", "pkg", "install", p], cwd=app_dir)
app_file = os.path.join(app_dir, "app.yaml")
with open(app_file) as f:
app_yaml = yaml.load(f)
libraries = {}
for pkg in packages:
pkg = pkg.split("/")[1]
libraries[pkg] = {
'gitVersion': {
'commitSha': 'fake',
'refSpec': 'fake'
},
'name': pkg,
'registry': "kubeflow"
}
app_yaml['libraries'] = libraries
with open(app_file, "w") as f:
yaml.dump(app_yaml, f)
# Create vendor directory with a symlink to the src
# so that we use the code at the desired commit.
target_dir = os.path.join(app_dir, "vendor", "kubeflow")
REPO_ORG = "kubeflow"
REPO_NAME = "kubeflow"
REGISTRY_PATH = "kubeflow"
source = os.path.join(dir, "src", REPO_ORG, REPO_NAME, REGISTRY_PATH)
logging.info("Creating link %s -> %s", target_dir, source)
os.symlink(source, target_dir)
return app_dir