def get_grouped_values(self, col_name, group_by): if not col_name in self.keys: errors.internal_error( "col_name '{}' not in MetricSet".format(col_name)) values = [rec[col_name] for rec in self.records] return values
def gen_runs(self, target, data, model, mtype, stype, runs, nodes): # called 7x3x3x4x5x4 = 5040 times args = "" mopt = "" script = "code/miniMnist.py" if mtype == "multi": mopt = " --multi" script = "code/multi_commands.txt" elif mtype == "hp-args": args = "--lr=[.01, .03, .05] --optimizer=[sgd, adam] --mlp-units=[64, 100, 128, 256]" elif mtype == "hp-file": mopt = " --hp-config=code/miniSweeps.yaml" elif mtype != "single": errors.internal_error("unrecognized mtype={}".format(mtype)) sopt = " --search-type=" + stype if stype != "none" else "" nopt = " --nodes=" + str(nodes) if nodes else "" ropt = " --runs=" + str(runs) logs_dir = "{}/{}.data_{}.model_{}.{}.{}.{}.{}".format(SUBMIT_LOGS_DIR, target, data, model, mtype, stype, runs, nodes) fake_submit = True cmd = "xt run --target={}{}{}{}{} --data-action={} --model-action={} --submit-logs={} --fake-submit={} {} {}".format(target, ropt, nopt, mopt, sopt, data, model, logs_dir, fake_submit, script, args) self.all_cmds.append( {"cmd": cmd, "logs_dir": logs_dir} )
def decorator_hidden(func): @functools.wraps(func) def wrapper_hidden(*args, **kwargs): return func(*args, **kwargs) if debug_decorators: console.print("hidden decorator called, name=", name, ", func=", func.__name__) global current_cmd_info if not current_cmd_info: errors.internal_error( "@hidden decorators must be followed by a single @command decorator" ) # a hidden is really just a hidden option type_name = type if isinstance(type, str) else type.__name__ option_info = { "name": name, "hidden": True, "type": type_name, "default": default, "help": help } #current_cmd_info["hiddens"].append(hidden_info) update_or_insert_argument(current_cmd_info, "options", option_info) return wrapper_hidden
def decorator_clone(func): @functools.wraps(func) def wrapper_clone(*args, **kwargs): return func(*args, **kwargs) if debug_decorators: console.print("clone decorator called, source=", source, ", func=", func.__name__) global current_cmd_info, root_cmd_info if not current_cmd_info: errors.internal_error( "@clone decorators must be followed by a single @command or @root decorator" ) source_cmd_info = get_command_by_words(source.split("_")) if arguments: current_cmd_info["arguments"] += source_cmd_info["arguments"] if options: current_cmd_info["options"] += source_cmd_info["options"] return wrapper_clone
def decorator_flag(func): @functools.wraps(func) def wrapper_flag(*args, **kwargs): return func(*args, **kwargs) if debug_decorators: console.print("flag decorator called, name=", name, ", func=", func.__name__) global current_cmd_info, root_cmd_info # a flag is really just a type=flag option option_info = { "name": name, "hidden": False, "type": "flag", "multiple": False, "default": default, "help": help } if not current_cmd_info: errors.internal_error( "@flag decorators must be followed by a single @command or @root decorator" ) update_or_insert_argument(current_cmd_info, "options", option_info) return wrapper_flag
def decorator_option(func): @functools.wraps(func) def wrapper_option(*args, **kwargs): return func(*args, **kwargs) if debug_decorators: console.print("option decorator called, name=", name, ", func=", func.__name__) global current_cmd_info if not current_cmd_info: errors.internal_error( "@option decorators must be followed by a single @command decorator" ) type_name = type if isinstance(type, str) else type.__name__ option_info = { "name": name, "hidden": False, "required": required, "type": type_name, "multiple": multiple, "default": default, "values": values, "help": help } #current_cmd_info["options"].append(option_info) update_or_insert_argument(current_cmd_info, "options", option_info) return wrapper_option
def decorator_keyword_arg(func): @functools.wraps(func) def wrapper_keyword_arg(*args, **kwargs): return func(*args, **kwargs) if debug_decorators: console.print("keyword_arg decorator called, name=", name, ", func=", func.__name__) global current_cmd_info if not current_cmd_info: errors.internal_error( "@keyword_arg decorators must be followed by a single @command decorator" ) type_name = type if isinstance(type, str) else type.__name__ arg_info = { "name": name, "keywords": keywords, "required": required, "type": type_name, "help": help, "default": default } #current_cmd_info["keyword_args"].insert(0, arg_info) update_or_insert_argument(current_cmd_info, "arguments", arg_info) return wrapper_keyword_arg
def decorator_command(func): @functools.wraps(func) def wrapper_command(*args, **kwargs): return func(*args, **kwargs) # begin actual decorater processing global first_command if first_command: first_command = False # console.diag("processing first cmd decorator") #console.print("first command...") if name: cmd_name = name else: cmd_name = func.__name__.replace("_", " ") if debug_decorators: console.print("command decorator called, func=", func.__name__) dd = commands for name_part in cmd_name.split(" "): if name_part not in dd: dd[name_part] = {} dd = dd[name_part] cmd_info = { "name": cmd_name, "options_before_args": options_before_args, "keyword_optional": keyword_optional, "pass_by_args": pass_by_args, "group": group, "func": func, "arguments": [], "options": [], "examples": [], "faqs": [], "hidden": False, "see_alsos": [], "kwgroup": kwgroup, "kwhelp": kwhelp, "help": help } dd[""] = cmd_info if keyword_optional: # only 1 command can use this if "" in commands: errors.internal_error( "processing command decoration for '{}'; only 1 command can use 'keyword_optional'" .format(func.__name__)) commands[""] = cmd_info global current_cmd_info current_cmd_info = cmd_info # end actual decorater processing return wrapper_command
def gen_name_help_aligned(self, items, prefix="", separator=" ", name_field="name", help_prefix="", include_type=False, spacer=" ", grouping_enabled=True): max_name_len = max( len(fi[name_field]) for fi in items if name_field in fi) text = "" kwgroups_seen = {} if include_type: max_type_len = max(len(fi["type"]) for fi in items) else: max_type_len = 0 for fi in items: if not "hidden" in fi or not fi["hidden"]: name = fi[name_field] help_text = fi["help"] if separator: name = name.replace("_", separator) if grouping_enabled and "kwgroup" in fi: kwgroup = fi["kwgroup"] if kwgroup: if kwgroup in kwgroups_seen: continue kwgroups_seen[kwgroup] = 1 name = kwgroup help_text = fi["kwhelp"] if not help_text: errors.internal_error( "kwhelp must be defined for this first kwgroup member: " + fi[name_field]) if include_type: fmt = " " + prefix + "{:<" + str( max_name_len) + "} {:<" + str( max_type_len) + "} {}" text += fmt.format(name, fi["type"], help_prefix + fi["help"]) + "\n" else: fmt = " " + prefix + "{:<" + str(max_name_len) + "} {}" text += fmt.format(name, help_prefix + help_text) + "\n" if "keywords" in fi: text += fmt.format( "", "--> choose one: " + ", ".join(fi["keywords"])) + "\n\n" return text
def detach(self, callback): index = None with self.lock: index = self.callbacks.index(callback) if index > -1: del self.callbacks[index] del self.acallbacks[index] else: errors.internal_error("could not find callback to detach") return index
def cleanup(): set_test_group("cleanup...") file_utils.ensure_dir_deleted("upload_testing") file_utils.ensure_dir_deleted("download_testing") # check for errors in runs text = xt("xt list runs --status=error", capture_output=True) # print("\nruns with errors:") # print(text) if not "no matching runs found" in text[0]: errors.internal_error("quick-test: above 'list runs' contains errors")
def merge_internal_xt_config(internal_config_text): ''' merge the fn_internal_config into the default xt config file. ''' res_dir = get_resource_dir() # write the internal config text as file in the resources directory and load/validate fn_internal_config = os.path.join(res_dir, constants.FN_INTERNAL_CONFIG) file_utils.zap_file(fn_internal_config) file_utils.write_text_file(fn_internal_config, internal_config_text) file_utils.make_readonly(fn_internal_config) internal_config = load_and_validate_config(fn_internal_config, validate_as_default=False) # first, see if the "orig" version of the default file is present fn_orig_default = os.path.abspath( os.path.join(res_dir, constants.FN_ORIG_DEFAULT)) fn_default = fn_orig_default # if not found, fallback to the normal default version if not os.path.exists(fn_default): fn_default = get_default_config_path() # the default_config.yaml file is required if not os.path.exists(fn_default): errors.internal_error( "missing default config file: {}".format(fn_default)) # preserve normal default as "orig" name shutil.copyfile(fn_default, fn_orig_default) file_utils.make_readonly(fn_orig_default) default_config = load_and_validate_config(fn_default, validate_as_default=True) # merge internal config into the default config merge_configs(default_config, internal_config) # remove old default file (fn_default) fn_default = get_default_config_path() file_utils.zap_file(fn_default) # save new merged version (fn_default) text = pretty_yaml_dump(default_config.data) file_utils.write_text_file(fn_default, text) # make file readonly file_utils.make_readonly(fn_default)
def apply_runset_file(self, args, fn): #utils.debug_break() fn = os.path.abspath(fn) with open(fn, "rt") as infile: yd = yaml.safe_load(infile) if not constants.HPARAM_RUNSET in yd: errors.internal_error("found runset file without {} property: {}".format(constants.HPARAM_RUNSET, fn)) print("applying runset file to args: {}".format(fn)) hd = yd[constants.HPARAM_RUNSET ] for prop, val in hd.items(): prop = prop.replace("-", "_") setattr(args, prop, val)
def process_option_value(self, opt_name, opt_type, value, values): if values: found = self.match_keyword(value, values) if not found: self.syntax_error( "Value for option {} not recognized: {}, must be one of: {}" .format(opt_name, value, ", ".join(values))) value = found elif opt_type == "flag": value = self.parse_flag_value(opt_name, value) elif opt_type == "int": value = int(value) elif opt_type == "float": value = float(value) elif opt_type == "bool": value = value.lower() if value in ["true", "1"]: value = True elif value in ["false", "0"]: value = False else: self.syntax_error("Illegal value for boolean option: " + str(value)) elif opt_type == "prop_op_value": value = self.process_prop_op_value(value) elif opt_type == "str_list": if not isinstance(value, list): value = [value] elif opt_type == "named_arg_list": if not isinstance(value, list): value = [value] value = self.convert_str_list_to_arg_dict(value) elif opt_type == "int_list": if not isinstance(value, list): value = [value] elif opt_type == "num_list": if not isinstance(value, list): value = [value] elif opt_type == "str": value = self.expand_system_values(value) else: errors.internal_error( "unrecognized option type: {}".format(opt_type)) return value
def __init__(self, mongo_conn_str, run_cache_dir): if not mongo_conn_str: errors.internal_error( "Cannot initialize MongoDB() with a empty mongo_conn_str") self.mongo_conn_str = mongo_conn_str self.run_cache_dir = run_cache_dir self.mongo_client = None self.mongo_db = None # keep a count of how many retryable errors we have encountered self.retry_errors = 0 self.run_cache_dir = os.path.expanduser( run_cache_dir) if run_cache_dir else None # initialize mondo-db now self.init_mongo_db_connection()
def get_simple_status(self, status): # translates an BATCH status to a simple status (queued, running, completed) queued = ["queued"] running = ["running"] completed = ["completed"] if status in queued: ss = "queued" elif status in running: ss = "running" elif status in completed: ss = "completed" else: errors.internal_error( "unexpected Pool status value: {}".format(status)) return ss
def read_config(self, fn=None): if fn is None: fn = get_default_config_path() self.fn = fn if not os.path.exists(fn): errors.internal_error("missing default_config file: " + fn) # read config file try: with open(fn, "rt") as file: config = yaml.safe_load(file) # , Loader=yaml.FullLoader) except BaseException as ex: logger.exception("Error in read_config, ex={}".format(ex)) raise Exception( "The config file '{}' is not valid YAML, error: {}".format( fn, ex)) return config
def add_column(self, col_name, col_values): if col_name in self.keys: errors.internal_error( "col_name '{}' already in MetricSet".format(col_name)) if not isinstance(col_values, (list, tuple)): # convert single value to a list col_values = col_values * len(self.records) if len(col_values) != list(self.records): errors.internal_error( "count of values for col_name '{}' is {}, but count of values in MetricSet is {}" .format(col_name, len(col_values), len(self.records))) # add to keys self.keys.append(col_name) # add each value for record, value in zip(self.records, col_values): record[col_name] in value
def mark_child_run_completed(self, entry): console.print("marking child run complete: entry={}".format(entry)) run_index = entry["run_index"] # optional assert ar = self._get_job_property("active_runs") ent = utils.find_by_property(ar, "run_index", run_index) if ent["status"] == constants.COMPLETED: errors.internal_error( "mark_child_run_completed: run already marked completed: {}". format(ent)) fd = {"_id": self.job_id, "active_runs.run_index": run_index} # mark entry as constants.COMPLETED cmd = lambda: self.mongo.mongo_db["__jobs__"].find_and_modify( fd, update={"$set": { "active_runs.$.status": constants.COMPLETED }}) self.mongo.mongo_with_retries("mark_child_run_completed", cmd)
def decorator_example(func): @functools.wraps(func) def wrapper_example(*args, **kwargs): return func(*args, **kwargs) if debug_decorators: console.print("example decorator called, name=", name, ", func=", func.__name__) global current_cmd_info, root_cmd_info example_info = {"text": text, "task": task, "image": image} if not current_cmd_info: errors.internal_error( "@example decorators must be followed by a single @command or @root decorator" ) #console.print("setting example name=", name) current_cmd_info["examples"].insert(0, example_info) return wrapper_example
def get_fn_run(self, args): # find first non-option at end of cmd to mark end of "fn_run" fn_run = "" #console.print("get_fn_run: args=", args) if not args: errors.internal_error("get_fn_run: args cannot be empty") if len(args) >= 2: if args[0] == "run": fn_run = os.path.abspath(args[1]) elif args[0] == "python": # skip over python options index = 1 while index < len(args) and args[index].startswith("-"): index += 1 if index < len(args): fn_run = os.path.abspath(args[index]) #console.print("fn_run=", fn_run) return fn_run
def decorator_see_also(func): @functools.wraps(func) def wrapper_see_also(*args, **kwargs): return func(*args, **kwargs) if debug_decorators: console.print("see_also decorator called, name=", name, ", func=", func.__name__) global current_cmd_info, root_cmd_info see_also_info = {"text": text, "page_path": page_path} if not current_cmd_info: errors.internal_error( "@see_also decorators must be followed by a single @command or @root decorator" ) #console.print("setting see_also name=", name) current_cmd_info["see_alsos"].insert(0, see_also_info) return wrapper_see_also
def decorator_faq(func): @functools.wraps(func) def wrapper_faq(*args, **kwargs): return func(*args, **kwargs) if debug_decorators: console.print("faq decorator called, name=", name, ", func=", func.__name__) global current_cmd_info, root_cmd_info faq_info = {"question": question, "answer": answer} if not current_cmd_info: errors.internal_error( "@faq decorators must be followed by a single @command or @root decorator" ) #console.print("setting faq name=", name) current_cmd_info["faqs"].insert(0, faq_info) return wrapper_faq
def get_simple_status(self, status): # translates an AML status to a simple status (queued, running, completed) queued = [ "NotStarted", "Starting", "Provisioning", "Preparing", "Queued" ] running = ["Running"] completed = [ "Finalizing", "CancelRequested", "Completed", "Failed", "Canceled", "NotResponding" ] if status in queued: ss = "queued" elif status in running: ss = "running" elif status in completed: ss = "completed" else: errors.internal_error( "unexpected Auzre ML status value: {}".format(status)) return ss
def schema_error(self, msg): full_msg = "Error in schema definition: {} (schema: {})".format( msg, self.schema_fn) errors.internal_error(full_msg)
def fixup_user_cmd(flat_cmd, for_windows, is_local, run_info, concurrent, hold): ''' Macros supported: $HOME - same as "~/" $IP_ADDR - ip address of target box $REGISTRY - the name of the user's docker registry (from config file) $MR_INDEX - the child run index modulo the max-runs for the box $HOLD - the True/False value of --hold option $CURRENT_CONDA_ENV - the name of the currenly active conda virtual environment Other updates: - translate between "$(pwd)" and "%cd", as appropriate for "for_windows" ''' mr_index = None docker_server = None if run_info: docker_server = run_info.context.docker_server run_name = run_info.run_name if "." in run_name: # compute mr_index (the child run index, modulo the max-runs value) # variations: run23, run23.2, exper.run123, exper.run123.3 right_part = run_name.split(".")[-1] if not right_part.startswith("run"): run_index = int(right_part) mr_index = str((run_index-1) % concurrent) console.print("run_index=", run_index, ", mr_index=", mr_index) home_dir = os.path.expanduser("~/") ip_addr = pc_utils.get_ip_address() if is_local else "" conda = pc_utils.get_conda_env() if not conda: conda = "py36" # reasonable default # unconditional: we always have this info flat_cmd = flat_cmd.replace("$CURRENT_CONDA_ENV", conda) flat_cmd = flat_cmd.replace("$HOLD", str(hold)) if "$REGISTRY" in flat_cmd: flat_cmd = flat_cmd.replace("$REGISTRY", docker_server) if "$MR_INDEX" in flat_cmd: flat_cmd = flat_cmd.replace("$MR_INDEX", mr_index) if "$HOME" in flat_cmd: if not is_local: errors.internal_error("cannot expand $HOME for remote box") flat_cmd = flat_cmd.replace("$HOME", home_dir) if "$IP_ADDR" in flat_cmd: # if not is_local: # errors.internal_error("cannot expand $IP_ADDR for remote box") ip_addr = "'" + ip_addr + "'" # surround it with single quotes flat_cmd = flat_cmd.replace("$IP_ADDR", ip_addr) if for_windows: flat_cmd = flat_cmd.replace("$(pwd)", "%cd%") else: flat_cmd = flat_cmd.replace("%cd%", "$(pwd)") # add "-u" unbuffered flag to python, if this marked as the RUN CMD if "%*" in flat_cmd or "$*" in flat_cmd: if not "-u" in flat_cmd: if flat_cmd.startswith("python "): flat_cmd = "python -u " + flat_cmd[7:] elif flat_cmd.startswith("python3 "): flat_cmd = "python3 -u " + flat_cmd[8:] return flat_cmd
def ensure_connected(self): if not self.conn: errors.internal_error("XTClient is not connected")
def parse_option_value(self, name, opt_type, keywords, found_equals, tok, scanner): if opt_type == "str_list": value, tok = self.parse_string_list(tok, scanner) if len(value) == 0 and required: self.syntax_error("missing value for required option: " + name) elif opt_type == "num_list": value, tok = self.parse_num_list(tok, scanner) if len(value) == 0 and required: self.syntax_error("missing value for required option: " + name) elif opt_type == "int_list": value, tok = self.parse_int_list(tok, scanner) if len(value) == 0 and required: self.syntax_error("missing value for required option: " + name) elif opt_type == "tag_list": value, tok = self.parse_tag_list(tok, scanner) if len(value) == 0 and required: self.syntax_error("missing value for required option: " + name) elif opt_type == "prop_op_value": value, tok = self.parse_prop_op_value_list(tok, scanner) if len(value) == 0 and required: self.syntax_error("missing value for required option: " + name) elif opt_type == "named_arg_list": value, tok = self.parse_string_list(tok, scanner) if not isinstance(value, list): value = [value] value = self.convert_str_list_to_arg_dict(value) elif opt_type == "flag": if found_equals: value = tok tok = scanner.scan() value = self.parse_flag_value(name, value) else: value = 1 else: # its a simple value value = tok tok = scanner.scan() if opt_type == "int": value = int(value) elif opt_type == "float": value = float(value) elif opt_type == "bool": value = value.lower() if value in ["true", "1"]: value = True elif value in ["false", "0"]: value = False else: self.syntax_error("Illegal value for boolean option: " + str(value)) elif opt_type == "str": value = self.expand_system_values(value) else: errors.internal_error( "Unsupported opt_type={}".format(opt_type)) return value, tok
def does_file_exist(self, fn): container, path, wc_target = self._get_container_path_target(fn) if wc_target: errors.internal_error("wildcard cannot be specified here") return self.store.provider.does_blob_exist(container, path)