def help_topics(self, topic, browse, prefix="topics", title="help topics"):
# build list of help topics from xtlib/help_topics directory
topics_dir = os.path.join(file_utils.get_xtlib_dir(), "help_topics",
prefix)
if not os.path.isdir(topics_dir):
errors.env_error("Missing help topics dir: {}".format(topics_dir))
topic_files, _ = file_utils.get_local_filenames(topics_dir)
# build a map from topic names to the files
topic_map = {file_utils.root_name(fn): fn for fn in topic_files}
if not topic:
console.print("available {}:".format(title))
keys = list(topic_map.keys())
keys.sort()
for topic_name in keys:
console.print(" {}".format(topic_name))
console.print()
console.print(
"To display a help topic, use 'xt help topic <topic name>'")
else:
# print a specific topic
topic_low = topic.lower()
if not topic_low in topic_map:
errors.general_error(
"help topic not found: {}".format(topic_low))
text = file_utils.read_text_file(topic_map[topic_low])
print(text)
def create_demo(self, destination, response, overwrite):
'''
This command will removed the specified destination directory if it exists (prompting the user for approval).
Specifying the current directory as the destination will produce an error.
'''
# set up from_dir
from_dir = file_utils.get_xtlib_dir() + "/demo_files"
# set up dest_dir
dest_dir = destination
if not dest_dir:
errors.syntax_error("An output directory must be specified")
create = True
console.print("creating demo files at: {}".format(
os.path.abspath(dest_dir)))
if os.path.exists(dest_dir):
answer = pc_utils.input_response(
"'{}' already exists; OK to delete? (y/n): ".format(dest_dir),
response)
if answer != "y":
create = False
if create:
file_utils.ensure_dir_deleted(dest_dir)
shutil.copytree(from_dir, dest_dir)
#file_utils.copy_tree(from_dir, dest_dir)
if not self.store.does_workspace_exist("xt-demo"):
# import xt-demo workspace from archive file
console.print(
"importing xt-demo workspace (usually takes about 30 seconds)"
)
impl_storage_api = ImplStorageApi(self.config, self.store)
fn_archive = os.path.join(file_utils.get_xtlib_dir(),
"demo_files", "xt-demo-archive.zip")
impl_storage_api.import_workspace(fn_archive,
"xt-demo",
"xtd",
overwrite=overwrite,
show_output=False)
def overwrite_default_config():
default_config_path = os.path.join(get_resource_dir(),
constants.FN_DEFAULT_CONFIG)
if is_default_config_present():
file_utils.zap_file(default_config_path)
res_dir = get_resource_dir()
file_utils.ensure_dir_exists(res_dir)
fn_source = os.path.join(file_utils.get_xtlib_dir(), "helpers",
constants.FN_DEFAULT_CONFIG)
shutil.copyfile(fn_source, default_config_path)
def load_and_validate_config(fn, validate_as_default):
# load the config file
config = XTConfig(fn=fn, create_if_needed=False)
# load the validation schema
fn_schema = os.path.join(file_utils.get_xtlib_dir(), "helpers",
"xt_config_schema.yaml")
schema = load_yaml(fn_schema)
# validate the config file
validator = Validator()
validator.validate(schema, fn_schema, config.data, config.fn,
validate_as_default)
return config
def get_default_config_path():
'''
always call this function to find the "default_config.yaml" file.
calling this ensures that the file has been copied from its package location.
'''
res_dir = get_resource_dir()
fn = os.path.join(res_dir, constants.FN_DEFAULT_CONFIG)
if not os.path.exists(fn):
# copy it from its helpers dir in the installed package (or dev directory)
file_utils.ensure_dir_exists(res_dir)
fn_source = os.path.join(file_utils.get_xtlib_dir(), "helpers",
constants.FN_DEFAULT_CONFIG)
shutil.copyfile(fn_source, fn)
# make file readonly
file_utils.make_readonly(fn)
return fn
def snapshot_all_code(self, snapshot_dir, cmd_parts, args):
'''
make local snapshot of each code_dir (and xtlib, if needed)
'''
code_dirs = args["code_dirs"]
xtlib_capture = args["xtlib_upload"]
code_omit = args["code_omit"]
script_dir = None
code_upload = args["code_upload"]
# this step should always be done so that script_dir is removed from cmd_parts
script_dir = self.remove_script_dir_from_parts(cmd_parts)
if code_upload:
for i, code_dir in enumerate(code_dirs):
# fixup "$scriptdir" relative paths
if "$scriptdir" in code_dir:
code_dir = code_dir.replace("$scriptdir", script_dir)
if "::" in code_dir:
code_dir, dest_dir = code_dir.split("::")
else:
dest_dir = "."
self.make_local_snapshot(snapshot_dir, code_dir, dest_dir, code_omit)
else:
script_dir = snapshot_dir
if xtlib_capture:
# copy XTLIB directory to "xtlib" subdir of temp
xtlib_dir = file_utils.get_xtlib_dir()
dest_dir = snapshot_dir + "/xtlib"
file_utils.ensure_dir_deleted(dest_dir)
# don't copy the "demo_files" directory
shutil.copytree(xtlib_dir, dest_dir, ignore=shutil.ignore_patterns("demo_files"))
console.diag("after create local snapshot")
return script_dir
def test():
fn_schema = os.path.join(file_utils.get_xtlib_dir(), "helpers",
"xt_config_schema.yaml")
schema = file_utils.load_yaml(fn_schema)
# a good first test: the default config file!
fn_to_validate = get_default_config_path()
default_config = file_utils.load_yaml(fn_to_validate)
local_config = file_utils.load_yaml("xt_config.yaml")
qt_config = file_utils.load_yaml("../quick-test/xt_config.yaml")
validator = Validator()
started = time.time()
validator.validate(schema, default_config, True, "default config")
validator.validate(schema, local_config, False, "local_config")
validator.validate(schema, qt_config, False, "qt_config")
elapsed = time.time() - started
print("elapsed time: {:.4f} secs".format(elapsed))
def adjust_run_commands(self, job_id, job_runs, using_hp, experiment,
service_type, snapshot_dir, args):
'''
This method is called to allow the backend to inject needed shell commands before the user cmd. At the
time this is called, files can still be added to snapshot_dir.
'''
store_data_dir, data_action, data_writable, store_model_dir, model_action, model_writable, \
storage_name, storage_key = self.get_action_args(args)
# local or POOL of vm's
fn_wrapped = None # we use same script for each box (but with different ARGS)
username = args["username"]
for i, box_runs in enumerate(job_runs):
# wrap the user commands in FIRST RUN of each box (apply data/model actions)
br = box_runs[0]
box_info = br["box_info"]
actions = ["data", "model"]
run_name = br["run_name"]
is_windows = False
node_id = utils.node_id(i)
run_specs = br["run_specs"]
cmd_parts = run_specs["cmd_parts"]
if not fn_wrapped:
# just wrap the user cmd once (shared by all boxes/nodes)
assert cmd_parts[0] == "python"
assert cmd_parts[1] == "-u"
assert len(cmd_parts[2]) > 0
# update the target_fn (might have been switched to the xt controller)
target_fn = cmd_parts[2]
arg_parts = cmd_parts[3:]
setup = self.config.get_setup_from_target_def(self.compute_def)
# we only do this once (for the first box/job)
fn_wrapped = super().wrap_user_command(cmd_parts,
snapshot_dir,
store_data_dir,
data_action,
data_writable,
store_model_dir,
model_action,
model_writable,
storage_name,
storage_key,
actions,
is_windows=is_windows,
sudo_available=False,
username=username,
use_username=False,
install_blobfuse=True,
setup=setup,
change_dir=False,
args=args)
# AML wants a python script, so use our tiny python shim to run wrapped.sh
fn_shim = "aml_shim.py"
fn_from = file_utils.get_xtlib_dir() + "/backends/" + fn_shim
fn_to = snapshot_dir + "/" + fn_shim
shutil.copyfile(fn_from, fn_to)
# copy to submit-logs
utils.copy_to_submit_logs(args, fn_from)
# we update each box's command (passing RUN_NAME as arg to wrapped.sh)
script_part = "{} {} {}".format(os.path.basename(fn_wrapped),
node_id, run_name)
sh_parts = ['/bin/bash', '--login', script_part]
# pass sh_parts as a single argument to avoid wierd "arg": 1 problems with AML estimators
wrapped_parts = ["python", "-u", fn_shim, " ".join(sh_parts)]
run_specs["cmd_parts"] = wrapped_parts
def get_default_config_template_path():
return os.path.join(file_utils.get_xtlib_dir(), "helpers",
constants.FN_DEFAULT_TEMPLATE)