def get_grouped_values(self, col_name, group_by):
if not col_name in self.keys:
errors.internal_error(
"col_name '{}' not in MetricSet".format(col_name))
values = [rec[col_name] for rec in self.records]
return values
def gen_runs(self, target, data, model, mtype, stype, runs, nodes):
# called 7x3x3x4x5x4 = 5040 times
args = ""
mopt = ""
script = "code/miniMnist.py"
if mtype == "multi":
mopt = " --multi"
script = "code/multi_commands.txt"
elif mtype == "hp-args":
args = "--lr=[.01, .03, .05] --optimizer=[sgd, adam] --mlp-units=[64, 100, 128, 256]"
elif mtype == "hp-file":
mopt = " --hp-config=code/miniSweeps.yaml"
elif mtype != "single":
errors.internal_error("unrecognized mtype={}".format(mtype))
sopt = " --search-type=" + stype if stype != "none" else ""
nopt = " --nodes=" + str(nodes) if nodes else ""
ropt = " --runs=" + str(runs)
logs_dir = "{}/{}.data_{}.model_{}.{}.{}.{}.{}".format(SUBMIT_LOGS_DIR, target, data, model, mtype, stype, runs, nodes)
fake_submit = True
cmd = "xt run --target={}{}{}{}{} --data-action={} --model-action={} --submit-logs={} --fake-submit={} {} {}".format(target, ropt, nopt, mopt, sopt, data, model, logs_dir, fake_submit, script, args)
self.all_cmds.append( {"cmd": cmd, "logs_dir": logs_dir} )
def decorator_hidden(func):
@functools.wraps(func)
def wrapper_hidden(*args, **kwargs):
return func(*args, **kwargs)
if debug_decorators:
console.print("hidden decorator called, name=", name, ", func=",
func.__name__)
global current_cmd_info
if not current_cmd_info:
errors.internal_error(
"@hidden decorators must be followed by a single @command decorator"
)
# a hidden is really just a hidden option
type_name = type if isinstance(type, str) else type.__name__
option_info = {
"name": name,
"hidden": True,
"type": type_name,
"default": default,
"help": help
}
#current_cmd_info["hiddens"].append(hidden_info)
update_or_insert_argument(current_cmd_info, "options", option_info)
return wrapper_hidden
def decorator_clone(func):
@functools.wraps(func)
def wrapper_clone(*args, **kwargs):
return func(*args, **kwargs)
if debug_decorators:
console.print("clone decorator called, source=", source, ", func=",
func.__name__)
global current_cmd_info, root_cmd_info
if not current_cmd_info:
errors.internal_error(
"@clone decorators must be followed by a single @command or @root decorator"
)
source_cmd_info = get_command_by_words(source.split("_"))
if arguments:
current_cmd_info["arguments"] += source_cmd_info["arguments"]
if options:
current_cmd_info["options"] += source_cmd_info["options"]
return wrapper_clone
def decorator_flag(func):
@functools.wraps(func)
def wrapper_flag(*args, **kwargs):
return func(*args, **kwargs)
if debug_decorators:
console.print("flag decorator called, name=", name, ", func=",
func.__name__)
global current_cmd_info, root_cmd_info
# a flag is really just a type=flag option
option_info = {
"name": name,
"hidden": False,
"type": "flag",
"multiple": False,
"default": default,
"help": help
}
if not current_cmd_info:
errors.internal_error(
"@flag decorators must be followed by a single @command or @root decorator"
)
update_or_insert_argument(current_cmd_info, "options", option_info)
return wrapper_flag
def decorator_option(func):
@functools.wraps(func)
def wrapper_option(*args, **kwargs):
return func(*args, **kwargs)
if debug_decorators:
console.print("option decorator called, name=", name, ", func=",
func.__name__)
global current_cmd_info
if not current_cmd_info:
errors.internal_error(
"@option decorators must be followed by a single @command decorator"
)
type_name = type if isinstance(type, str) else type.__name__
option_info = {
"name": name,
"hidden": False,
"required": required,
"type": type_name,
"multiple": multiple,
"default": default,
"values": values,
"help": help
}
#current_cmd_info["options"].append(option_info)
update_or_insert_argument(current_cmd_info, "options", option_info)
return wrapper_option
def decorator_keyword_arg(func):
@functools.wraps(func)
def wrapper_keyword_arg(*args, **kwargs):
return func(*args, **kwargs)
if debug_decorators:
console.print("keyword_arg decorator called, name=", name,
", func=", func.__name__)
global current_cmd_info
if not current_cmd_info:
errors.internal_error(
"@keyword_arg decorators must be followed by a single @command decorator"
)
type_name = type if isinstance(type, str) else type.__name__
arg_info = {
"name": name,
"keywords": keywords,
"required": required,
"type": type_name,
"help": help,
"default": default
}
#current_cmd_info["keyword_args"].insert(0, arg_info)
update_or_insert_argument(current_cmd_info, "arguments", arg_info)
return wrapper_keyword_arg
def decorator_command(func):
@functools.wraps(func)
def wrapper_command(*args, **kwargs):
return func(*args, **kwargs)
# begin actual decorater processing
global first_command
if first_command:
first_command = False
# console.diag("processing first cmd decorator")
#console.print("first command...")
if name:
cmd_name = name
else:
cmd_name = func.__name__.replace("_", " ")
if debug_decorators:
console.print("command decorator called, func=", func.__name__)
dd = commands
for name_part in cmd_name.split(" "):
if name_part not in dd:
dd[name_part] = {}
dd = dd[name_part]
cmd_info = {
"name": cmd_name,
"options_before_args": options_before_args,
"keyword_optional": keyword_optional,
"pass_by_args": pass_by_args,
"group": group,
"func": func,
"arguments": [],
"options": [],
"examples": [],
"faqs": [],
"hidden": False,
"see_alsos": [],
"kwgroup": kwgroup,
"kwhelp": kwhelp,
"help": help
}
dd[""] = cmd_info
if keyword_optional:
# only 1 command can use this
if "" in commands:
errors.internal_error(
"processing command decoration for '{}'; only 1 command can use 'keyword_optional'"
.format(func.__name__))
commands[""] = cmd_info
global current_cmd_info
current_cmd_info = cmd_info
# end actual decorater processing
return wrapper_command
def gen_name_help_aligned(self,
items,
prefix="",
separator=" ",
name_field="name",
help_prefix="",
include_type=False,
spacer=" ",
grouping_enabled=True):
max_name_len = max(
len(fi[name_field]) for fi in items if name_field in fi)
text = ""
kwgroups_seen = {}
if include_type:
max_type_len = max(len(fi["type"]) for fi in items)
else:
max_type_len = 0
for fi in items:
if not "hidden" in fi or not fi["hidden"]:
name = fi[name_field]
help_text = fi["help"]
if separator:
name = name.replace("_", separator)
if grouping_enabled and "kwgroup" in fi:
kwgroup = fi["kwgroup"]
if kwgroup:
if kwgroup in kwgroups_seen:
continue
kwgroups_seen[kwgroup] = 1
name = kwgroup
help_text = fi["kwhelp"]
if not help_text:
errors.internal_error(
"kwhelp must be defined for this first kwgroup member: "
+ fi[name_field])
if include_type:
fmt = " " + prefix + "{:<" + str(
max_name_len) + "} {:<" + str(
max_type_len) + "} {}"
text += fmt.format(name, fi["type"],
help_prefix + fi["help"]) + "\n"
else:
fmt = " " + prefix + "{:<" + str(max_name_len) + "} {}"
text += fmt.format(name, help_prefix + help_text) + "\n"
if "keywords" in fi:
text += fmt.format(
"", "--> choose one: " +
", ".join(fi["keywords"])) + "\n\n"
return text
def detach(self, callback):
index = None
with self.lock:
index = self.callbacks.index(callback)
if index > -1:
del self.callbacks[index]
del self.acallbacks[index]
else:
errors.internal_error("could not find callback to detach")
return index
def cleanup():
set_test_group("cleanup...")
file_utils.ensure_dir_deleted("upload_testing")
file_utils.ensure_dir_deleted("download_testing")
# check for errors in runs
text = xt("xt list runs --status=error", capture_output=True)
# print("\nruns with errors:")
# print(text)
if not "no matching runs found" in text[0]:
errors.internal_error("quick-test: above 'list runs' contains errors")
def merge_internal_xt_config(internal_config_text):
'''
merge the fn_internal_config into the default xt config file.
'''
res_dir = get_resource_dir()
# write the internal config text as file in the resources directory and load/validate
fn_internal_config = os.path.join(res_dir, constants.FN_INTERNAL_CONFIG)
file_utils.zap_file(fn_internal_config)
file_utils.write_text_file(fn_internal_config, internal_config_text)
file_utils.make_readonly(fn_internal_config)
internal_config = load_and_validate_config(fn_internal_config,
validate_as_default=False)
# first, see if the "orig" version of the default file is present
fn_orig_default = os.path.abspath(
os.path.join(res_dir, constants.FN_ORIG_DEFAULT))
fn_default = fn_orig_default
# if not found, fallback to the normal default version
if not os.path.exists(fn_default):
fn_default = get_default_config_path()
# the default_config.yaml file is required
if not os.path.exists(fn_default):
errors.internal_error(
"missing default config file: {}".format(fn_default))
# preserve normal default as "orig" name
shutil.copyfile(fn_default, fn_orig_default)
file_utils.make_readonly(fn_orig_default)
default_config = load_and_validate_config(fn_default,
validate_as_default=True)
# merge internal config into the default config
merge_configs(default_config, internal_config)
# remove old default file (fn_default)
fn_default = get_default_config_path()
file_utils.zap_file(fn_default)
# save new merged version (fn_default)
text = pretty_yaml_dump(default_config.data)
file_utils.write_text_file(fn_default, text)
# make file readonly
file_utils.make_readonly(fn_default)
def apply_runset_file(self, args, fn):
#utils.debug_break()
fn = os.path.abspath(fn)
with open(fn, "rt") as infile:
yd = yaml.safe_load(infile)
if not constants.HPARAM_RUNSET in yd:
errors.internal_error("found runset file without {} property: {}".format(constants.HPARAM_RUNSET, fn))
print("applying runset file to args: {}".format(fn))
hd = yd[constants.HPARAM_RUNSET ]
for prop, val in hd.items():
prop = prop.replace("-", "_")
setattr(args, prop, val)
def process_option_value(self, opt_name, opt_type, value, values):
if values:
found = self.match_keyword(value, values)
if not found:
self.syntax_error(
"Value for option {} not recognized: {}, must be one of: {}"
.format(opt_name, value, ", ".join(values)))
value = found
elif opt_type == "flag":
value = self.parse_flag_value(opt_name, value)
elif opt_type == "int":
value = int(value)
elif opt_type == "float":
value = float(value)
elif opt_type == "bool":
value = value.lower()
if value in ["true", "1"]:
value = True
elif value in ["false", "0"]:
value = False
else:
self.syntax_error("Illegal value for boolean option: " +
str(value))
elif opt_type == "prop_op_value":
value = self.process_prop_op_value(value)
elif opt_type == "str_list":
if not isinstance(value, list):
value = [value]
elif opt_type == "named_arg_list":
if not isinstance(value, list):
value = [value]
value = self.convert_str_list_to_arg_dict(value)
elif opt_type == "int_list":
if not isinstance(value, list):
value = [value]
elif opt_type == "num_list":
if not isinstance(value, list):
value = [value]
elif opt_type == "str":
value = self.expand_system_values(value)
else:
errors.internal_error(
"unrecognized option type: {}".format(opt_type))
return value
def __init__(self, mongo_conn_str, run_cache_dir):
if not mongo_conn_str:
errors.internal_error(
"Cannot initialize MongoDB() with a empty mongo_conn_str")
self.mongo_conn_str = mongo_conn_str
self.run_cache_dir = run_cache_dir
self.mongo_client = None
self.mongo_db = None
# keep a count of how many retryable errors we have encountered
self.retry_errors = 0
self.run_cache_dir = os.path.expanduser(
run_cache_dir) if run_cache_dir else None
# initialize mondo-db now
self.init_mongo_db_connection()
def get_simple_status(self, status):
# translates an BATCH status to a simple status (queued, running, completed)
queued = ["queued"]
running = ["running"]
completed = ["completed"]
if status in queued:
ss = "queued"
elif status in running:
ss = "running"
elif status in completed:
ss = "completed"
else:
errors.internal_error(
"unexpected Pool status value: {}".format(status))
return ss
def read_config(self, fn=None):
if fn is None:
fn = get_default_config_path()
self.fn = fn
if not os.path.exists(fn):
errors.internal_error("missing default_config file: " + fn)
# read config file
try:
with open(fn, "rt") as file:
config = yaml.safe_load(file) # , Loader=yaml.FullLoader)
except BaseException as ex:
logger.exception("Error in read_config, ex={}".format(ex))
raise Exception(
"The config file '{}' is not valid YAML, error: {}".format(
fn, ex))
return config
def add_column(self, col_name, col_values):
if col_name in self.keys:
errors.internal_error(
"col_name '{}' already in MetricSet".format(col_name))
if not isinstance(col_values, (list, tuple)):
# convert single value to a list
col_values = col_values * len(self.records)
if len(col_values) != list(self.records):
errors.internal_error(
"count of values for col_name '{}' is {}, but count of values in MetricSet is {}"
.format(col_name, len(col_values), len(self.records)))
# add to keys
self.keys.append(col_name)
# add each value
for record, value in zip(self.records, col_values):
record[col_name] in value
def mark_child_run_completed(self, entry):
console.print("marking child run complete: entry={}".format(entry))
run_index = entry["run_index"]
# optional assert
ar = self._get_job_property("active_runs")
ent = utils.find_by_property(ar, "run_index", run_index)
if ent["status"] == constants.COMPLETED:
errors.internal_error(
"mark_child_run_completed: run already marked completed: {}".
format(ent))
fd = {"_id": self.job_id, "active_runs.run_index": run_index}
# mark entry as constants.COMPLETED
cmd = lambda: self.mongo.mongo_db["__jobs__"].find_and_modify(
fd, update={"$set": {
"active_runs.$.status": constants.COMPLETED
}})
self.mongo.mongo_with_retries("mark_child_run_completed", cmd)
def decorator_example(func):
@functools.wraps(func)
def wrapper_example(*args, **kwargs):
return func(*args, **kwargs)
if debug_decorators:
console.print("example decorator called, name=", name, ", func=",
func.__name__)
global current_cmd_info, root_cmd_info
example_info = {"text": text, "task": task, "image": image}
if not current_cmd_info:
errors.internal_error(
"@example decorators must be followed by a single @command or @root decorator"
)
#console.print("setting example name=", name)
current_cmd_info["examples"].insert(0, example_info)
return wrapper_example
def get_fn_run(self, args):
# find first non-option at end of cmd to mark end of "fn_run"
fn_run = ""
#console.print("get_fn_run: args=", args)
if not args:
errors.internal_error("get_fn_run: args cannot be empty")
if len(args) >= 2:
if args[0] == "run":
fn_run = os.path.abspath(args[1])
elif args[0] == "python":
# skip over python options
index = 1
while index < len(args) and args[index].startswith("-"):
index += 1
if index < len(args):
fn_run = os.path.abspath(args[index])
#console.print("fn_run=", fn_run)
return fn_run
def decorator_see_also(func):
@functools.wraps(func)
def wrapper_see_also(*args, **kwargs):
return func(*args, **kwargs)
if debug_decorators:
console.print("see_also decorator called, name=", name, ", func=",
func.__name__)
global current_cmd_info, root_cmd_info
see_also_info = {"text": text, "page_path": page_path}
if not current_cmd_info:
errors.internal_error(
"@see_also decorators must be followed by a single @command or @root decorator"
)
#console.print("setting see_also name=", name)
current_cmd_info["see_alsos"].insert(0, see_also_info)
return wrapper_see_also
def decorator_faq(func):
@functools.wraps(func)
def wrapper_faq(*args, **kwargs):
return func(*args, **kwargs)
if debug_decorators:
console.print("faq decorator called, name=", name, ", func=",
func.__name__)
global current_cmd_info, root_cmd_info
faq_info = {"question": question, "answer": answer}
if not current_cmd_info:
errors.internal_error(
"@faq decorators must be followed by a single @command or @root decorator"
)
#console.print("setting faq name=", name)
current_cmd_info["faqs"].insert(0, faq_info)
return wrapper_faq
def get_simple_status(self, status):
# translates an AML status to a simple status (queued, running, completed)
queued = [
"NotStarted", "Starting", "Provisioning", "Preparing", "Queued"
]
running = ["Running"]
completed = [
"Finalizing", "CancelRequested", "Completed", "Failed", "Canceled",
"NotResponding"
]
if status in queued:
ss = "queued"
elif status in running:
ss = "running"
elif status in completed:
ss = "completed"
else:
errors.internal_error(
"unexpected Auzre ML status value: {}".format(status))
return ss
def schema_error(self, msg):
full_msg = "Error in schema definition: {} (schema: {})".format(
msg, self.schema_fn)
errors.internal_error(full_msg)
def fixup_user_cmd(flat_cmd, for_windows, is_local, run_info, concurrent, hold):
'''
Macros supported:
$HOME - same as "~/"
$IP_ADDR - ip address of target box
$REGISTRY - the name of the user's docker registry (from config file)
$MR_INDEX - the child run index modulo the max-runs for the box
$HOLD - the True/False value of --hold option
$CURRENT_CONDA_ENV - the name of the currenly active conda virtual environment
Other updates:
- translate between "$(pwd)" and "%cd", as appropriate for "for_windows"
'''
mr_index = None
docker_server = None
if run_info:
docker_server = run_info.context.docker_server
run_name = run_info.run_name
if "." in run_name:
# compute mr_index (the child run index, modulo the max-runs value)
# variations: run23, run23.2, exper.run123, exper.run123.3
right_part = run_name.split(".")[-1]
if not right_part.startswith("run"):
run_index = int(right_part)
mr_index = str((run_index-1) % concurrent)
console.print("run_index=", run_index, ", mr_index=", mr_index)
home_dir = os.path.expanduser("~/")
ip_addr = pc_utils.get_ip_address() if is_local else ""
conda = pc_utils.get_conda_env()
if not conda:
conda = "py36" # reasonable default
# unconditional: we always have this info
flat_cmd = flat_cmd.replace("$CURRENT_CONDA_ENV", conda)
flat_cmd = flat_cmd.replace("$HOLD", str(hold))
if "$REGISTRY" in flat_cmd:
flat_cmd = flat_cmd.replace("$REGISTRY", docker_server)
if "$MR_INDEX" in flat_cmd:
flat_cmd = flat_cmd.replace("$MR_INDEX", mr_index)
if "$HOME" in flat_cmd:
if not is_local:
errors.internal_error("cannot expand $HOME for remote box")
flat_cmd = flat_cmd.replace("$HOME", home_dir)
if "$IP_ADDR" in flat_cmd:
# if not is_local:
# errors.internal_error("cannot expand $IP_ADDR for remote box")
ip_addr = "'" + ip_addr + "'" # surround it with single quotes
flat_cmd = flat_cmd.replace("$IP_ADDR", ip_addr)
if for_windows:
flat_cmd = flat_cmd.replace("$(pwd)", "%cd%")
else:
flat_cmd = flat_cmd.replace("%cd%", "$(pwd)")
# add "-u" unbuffered flag to python, if this marked as the RUN CMD
if "%*" in flat_cmd or "$*" in flat_cmd:
if not "-u" in flat_cmd:
if flat_cmd.startswith("python "):
flat_cmd = "python -u " + flat_cmd[7:]
elif flat_cmd.startswith("python3 "):
flat_cmd = "python3 -u " + flat_cmd[8:]
return flat_cmd
def ensure_connected(self):
if not self.conn:
errors.internal_error("XTClient is not connected")
def parse_option_value(self, name, opt_type, keywords, found_equals, tok,
scanner):
if opt_type == "str_list":
value, tok = self.parse_string_list(tok, scanner)
if len(value) == 0 and required:
self.syntax_error("missing value for required option: " + name)
elif opt_type == "num_list":
value, tok = self.parse_num_list(tok, scanner)
if len(value) == 0 and required:
self.syntax_error("missing value for required option: " + name)
elif opt_type == "int_list":
value, tok = self.parse_int_list(tok, scanner)
if len(value) == 0 and required:
self.syntax_error("missing value for required option: " + name)
elif opt_type == "tag_list":
value, tok = self.parse_tag_list(tok, scanner)
if len(value) == 0 and required:
self.syntax_error("missing value for required option: " + name)
elif opt_type == "prop_op_value":
value, tok = self.parse_prop_op_value_list(tok, scanner)
if len(value) == 0 and required:
self.syntax_error("missing value for required option: " + name)
elif opt_type == "named_arg_list":
value, tok = self.parse_string_list(tok, scanner)
if not isinstance(value, list):
value = [value]
value = self.convert_str_list_to_arg_dict(value)
elif opt_type == "flag":
if found_equals:
value = tok
tok = scanner.scan()
value = self.parse_flag_value(name, value)
else:
value = 1
else:
# its a simple value
value = tok
tok = scanner.scan()
if opt_type == "int":
value = int(value)
elif opt_type == "float":
value = float(value)
elif opt_type == "bool":
value = value.lower()
if value in ["true", "1"]:
value = True
elif value in ["false", "0"]:
value = False
else:
self.syntax_error("Illegal value for boolean option: " +
str(value))
elif opt_type == "str":
value = self.expand_system_values(value)
else:
errors.internal_error(
"Unsupported opt_type={}".format(opt_type))
return value, tok
def does_file_exist(self, fn):
container, path, wc_target = self._get_container_path_target(fn)
if wc_target:
errors.internal_error("wildcard cannot be specified here")
return self.store.provider.does_blob_exist(container, path)
