def action(self):
"""Creates a new git repository based on an mlt template in the
current working directory.
"""
template_name = self.args["--template"]
template_repo = self.args["--template-repo"]
with git_helpers.clone_repo(template_repo) as temp_clone:
templates_directory = os.path.join(temp_clone, TEMPLATES_DIR,
template_name)
try:
shutil.copytree(templates_directory, self.app_name)
data = self._build_mlt_json()
with open(os.path.join(self.app_name, 'mlt.json'), 'w') as f:
json.dump(data, f, indent=2)
self._init_git_repo()
except OSError as exc:
if exc.errno == 17:
print(
"Directory '{}' already exists: delete before trying "
"to initialize new application".format(self.app_name))
else:
traceback.print_exc()
sys.exit(1)
def _setup_experiments_sa(self):
"""
Running the experiments template requires role/rolebindings setup
for the namespace. This method will create the namespace, and then
set the roles/rolebindings that we get from the experiments repo.
"""
experiments_repo = "https://github.com/IntelAI/experiments.git"
with clone_repo(experiments_repo) as temp_clone:
# get known version of experiments repo
command = ['git', 'checkout', get_experiments_version()]
self._launch_popen_call(command, cwd=temp_clone)
# get the sa.yaml file and then replace 'demo' with our namespace
example_dir = os.path.join(temp_clone, "examples/demo")
with open(os.path.join(example_dir, "sa.yaml"), 'r') as sa_file:
sa_yaml = sa_file.read()
sa_yaml = sa_yaml.replace("demo", self.namespace)
# write the yaml with the test namespace out to a new file
new_sa_file = os.path.join(temp_clone, example_dir,
"sa_{}.yaml".format(self.namespace))
with open(new_sa_file, 'w') as sa_file:
sa_file.write(sa_yaml)
# create namespace so that we can add roles
command = ['kubectl', 'create', 'ns', self.namespace]
self._launch_popen_call(command, stderr_is_not_okay=True)
# create roles/rolebindings using kubectl command
command = ['kubectl', 'create', '-f', new_sa_file]
self._launch_popen_call(command, stderr_is_not_okay=True)
def action(self):
"""lists templates available"""
template_repo = self.args["--template-repo"]
with git_helpers.clone_repo(template_repo) as temp_clone:
templates_directory = os.path.join(temp_clone, TEMPLATES_DIR)
templates = self._parse_templates(templates_directory)
print(
tabulate(templates,
headers=['Template', 'Description'],
tablefmt="simple"))
def test_clone_repo(contextmanager_mock, copy_tree_mock, is_git_repo_mock,
os_path_exists_mock, run_popen_mock, shutil_mock,
tempfile_mock, git_repo):
is_git_repo_mock.return_value = git_repo
os_path_exists_mock.return_value = True
with git_helpers.clone_repo('hello-world'):
# hack to get the contextmanager func called
pass
run_popen_mock.return_value.wait.assert_called_once()
if not git_repo:
copy_tree_mock.assert_called_once()
def test_local_templates(self):
"""
Tests creating a new template in a clone of mlt. Verifies that we can
specify the template-repo for mlt template list and see new template in
the list. Then uses mlt init to create an app with new template and
verifies that the app directory exists.
"""
# Create a git clone of mlt to use as a local template diretory
with git_helpers.clone_repo(project.basedir()) as temp_clone:
# Add a new test template to the local mlt template directory
templates_directory = os.path.join(
temp_clone, constants.TEMPLATES_DIR)
new_template_name = "test-local"
new_template_directory = os.path.join(templates_directory,
new_template_name)
os.mkdir(new_template_directory)
new_template_file = os.path.join(
new_template_directory, "README.md")
with open(new_template_file, "w") as f:
f.write("New local template for testing")
# call mlt template list and then check the output
output = self._call_template_list(temp_clone)
# template list should include our new test-local template
desired_template_output = """Template Description
------------------- ---------------------------------------------------------\
-----------------------------------------
experiments Runs hyperparameter experiments for a demo job.
hello-world A TensorFlow python HelloWorld example run through \
Kubernetes Jobs.
horovod A distributed model training using horovod and openmpi \
with added support for S3 storage.
pytorch Sample distributed application taken from \
http://pytorch.org/tutorials/intermediate/dist_tuto.html
pytorch-distributed A distributed PyTorch MNIST example run using the \
pytorch-operator.
tensorboard-bm A TensorBoard service in Kubernetes Bare Metal cluster.
tensorboard-gke A TensorBoard service in Google Kubernetes cluster.
test-local New local template for testing
tf-dist-mnist A distributed TensorFlow MNIST model which designates \
worker 0 as the chief.
tf-distributed A distributed TensorFlow matrix multiplication run \
through the TensorFlow Kubernetes Operator.
"""
assert output == desired_template_output
# init an app with this new template and verify that the app exists
self.init(template=new_template_name, template_repo=temp_clone)
assert os.path.isdir(self.project_dir)
assert os.path.isfile(os.path.join(
self.project_dir, "README.md"))
def action(self):
"""lists templates available"""
template_repo = self.args["--template-repo"]
with git_helpers.clone_repo(template_repo) as temp_clone:
templates_directory = os.path.join(temp_clone, TEMPLATES_DIR)
templates = self._parse_templates(templates_directory)
if not templates:
if git_helpers.is_git_repo(template_repo):
print("Please verify git is installed and setup "
"properly and you have read access to the repo.")
else:
print("Please make sure template repo directory exists "
"and you have read access to the directory.")
else:
print(
tabulate(templates,
headers=['Template', 'Description'],
tablefmt="simple"))
def action(self):
"""Creates a new git repository based on an mlt template in the
current working directory.
"""
template_name = self.args["--template"]
template_repo = self.args["--template-repo"]
skip_crd_check = self.args["--skip-crd-check"]
with git_helpers.clone_repo(template_repo) as temp_clone:
templates_directory = os.path.join(temp_clone,
constants.TEMPLATES_DIR,
template_name)
try:
# The template configs get pulled into the mlt.json file, so
# don't grab a copy of that in the app directory
copytree(templates_directory,
self.app_name,
ignore=ignore_patterns(constants.TEMPLATE_CONFIG))
# Get the template configs from the template and include them
# when building the mlt json file
param_file = os.path.join(templates_directory,
constants.TEMPLATE_CONFIG)
template_params = config_helpers.\
get_template_parameters_from_file(param_file)
if not skip_crd_check:
kubernetes_helpers.check_crds(app_name=self.app_name)
data = self._build_mlt_json(template_params)
with open(os.path.join(self.app_name, constants.MLT_CONFIG),
'w') as f:
json.dump(data, f, indent=2)
self._init_git_repo()
except OSError as exc:
if exc.errno == 17:
print(
"Directory '{}' already exists: delete before trying "
"to initialize new application".format(self.app_name))
else:
traceback.print_exc()
sys.exit(1)
def test_local_templates():
""" Tests creating a new template in a clone of mlt. Verifies that we can
specify the template-repo for mlt template list and see the new template in
the list. Then uses mlt init to create an app with our new template and
verifies that the app directory exists. """
# Create a git clone of mlt to use as a local template diretory
with git_helpers.clone_repo(project.basedir()) as temp_clone:
# Add a new test template to the local mlt template directory
templates_directory = os.path.join(temp_clone, constants.TEMPLATES_DIR)
new_template_name = "test-local"
new_template_directory = os.path.join(templates_directory,
new_template_name)
os.mkdir(new_template_directory)
new_template_file = os.path.join(new_template_directory, "README.md")
with open(new_template_file, "w") as f:
f.write("New local template for testing")
# call mlt template list and then check the output
output = call_template_list(temp_clone)
# template list should include our new test-local template
desired_template_output = """Template Description
------------------- --------------------------------------------------------------------------------------------------
hello-world A TensorFlow python HelloWorld example run through Kubernetes Jobs.
pytorch Sample distributed application taken from http://pytorch.org/tutorials/intermediate/dist_tuto.html
pytorch-distributed A distributed PyTorch MNIST example run using the pytorch-operator.
test-local New local template for testing
tf-dist-mnist A distributed TensorFlow MNIST model which designates worker 0 as the chief.
tf-distributed A distributed TensorFlow matrix multiplication run through the TensorFlow Kubernetes Operator.
"""
assert output == desired_template_output
# init an app with this new template and verify that the app exists
with create_work_dir() as workdir:
commands = CommandTester(workdir)
commands.init(template=new_template_name, template_repo=temp_clone)
app_directory = os.path.join(workdir, commands.app_name)
assert os.path.isdir(app_directory)
assert os.path.isfile(os.path.join(app_directory, "README.md"))
def action(self):
"""Creates a new git repository based on an mlt template in the
current working directory.
"""
template_name = self.args["--template"]
template_repo = self.args["--template-repo"]
skip_crd_check = self.args["--skip-crd-check"]
with git_helpers.clone_repo(template_repo) as temp_clone:
templates_directory = os.path.join(temp_clone,
constants.TEMPLATES_DIR,
template_name)
try:
# The template configs get pulled into the mlt.json file, so
# don't grab a copy of that in this app's directory
copytree(templates_directory,
self.app_name,
ignore=ignore_patterns(constants.TEMPLATE_CONFIG))
# Get the template configs from the template and include them
# when building the mlt json file
param_file = os.path.join(templates_directory,
constants.TEMPLATE_CONFIG)
template_params = config_helpers.\
get_template_parameters_from_file(param_file)
template_git_sha = git_helpers.get_latest_sha(
os.path.join(temp_clone, constants.TEMPLATES_DIR,
template_name))
if not skip_crd_check:
kubernetes_helpers.check_crds(app_name=self.app_name)
if self.args["--enable-sync"]:
if localhost_helpers.binary_path('ksync'):
# Syncthing uses '.stignore' to ignore files during
# sync we also don't want to upload unneeded local data
app_ignore_file = os.path.join(self.app_name,
".gitignore")
ksync_ignore_file = os.path.join(
self.app_name, ".stignore")
if self._check_update_yaml_for_sync():
copyfile(app_ignore_file, ksync_ignore_file)
with open(ksync_ignore_file, 'a+') as f:
f.write("\n.git/**")
else:
error_handling.throw_error(
"This app doesn't support syncing", 'yellow')
else:
error_handling.throw_error(
"ksync is not installed on localhost.", 'red')
data = self._build_mlt_json(template_params, template_git_sha)
# If the app has option for debugging failures, grab the
# Kubernetes debug wrapper file and put it in the app directory
if any(param["name"] == "debug_on_fail"
for param in template_params):
self._enable_debug_on_fail(temp_clone)
with open(os.path.join(self.app_name, constants.MLT_CONFIG),
'w') as f:
json.dump(data, f, indent=2)
self._init_git_repo()
except OSError as exc:
if exc.errno == 17:
error_msg = "Directory '{}' already exists: ".format(
self.app_name) + \
"delete before trying to initialize new application"
color = 'red'
else:
error_msg = traceback.format_exc()
color = None
error_handling.throw_error(error_msg, color)
def action(self):
"""Update the template instance with new template version
if template is updated """
if "template_name" not in self.config or \
"template_git_sha" not in self.config:
print("ERROR: mlt.json does not have either template_name "
"or template_git_sha. Template update is not possible.")
return
app_name = self.config["name"]
template_name = self.config["template_name"]
current_template_git_sha = self.config["template_git_sha"]
orig_project_backup_dir = self._get_backup_dir_name(app_name)
with git_helpers.clone_repo(self.template_repo) as temp_clone:
application_dir = os.getcwd()
clone_template_dir = os.path.join(temp_clone,
constants.TEMPLATES_DIR,
template_name)
if not os.path.exists(clone_template_dir):
print("Unable to update, template {} does "
"not exist in MLT git repo.".format(template_name))
return
latest_template_git_sha = \
git_helpers.get_latest_sha(clone_template_dir)
if current_template_git_sha == latest_template_git_sha:
print("Template is up to date, no need for update.")
else:
print("Template is not up to date, updating template...")
copy_tree(application_dir, orig_project_backup_dir)
os.chdir(temp_clone)
# create temp-branch using git sha from which template
# was initiated and clean un-tracked files
cmd = "git checkout -f {} -b temp-branch && git clean -f .". \
format(current_template_git_sha)
process_helpers.run_popen(cmd, shell=True)
# copy app dir content to temp clone template dir
copy_tree(application_dir, clone_template_dir)
# if there are any uncommitted changes to temp-branch,
# commit them otherwise 'pull' from master will fail.
output = process_helpers.run("git status".split(" "))
if "Your branch is up-to-date" not in output:
process_helpers.run("git add --all ".split(" "))
commit_command = "git commit --message 'temp-commit'"
process_helpers.run(commit_command.split(" "))
# merging latest template changes by pulling from master
# into temp-branch
try:
process_helpers.run("git pull origin master".split(" "),
raise_on_failure=True)
except CalledProcessError as e:
# When auto merge failed do not error out,
# let user review and fix conflicts
# for other errors exit
error_string = "Automatic merge failed; " \
"fix conflicts and then commit the result"
if error_string not in e.output:
error_handling.throw_error(e.output)
# copy content of clone template dir back to app dir
copy_tree(clone_template_dir, application_dir)
print("Latest template changes have merged using git, "
"please review changes for conflicts. ")
print("Backup directory path: {}".format(
os.path.abspath(orig_project_backup_dir)))
os.chdir(application_dir)