def run(args, path=None, quiet=False): """ Runs MMT with the given arguments in a given path """ # setup path if path == None: path = os.getcwd() # run the mmt script, wait and return. try: if not os.path.isfile(mmt_executable): raise FileNotFoundError proc = subprocess.Popen( [java_executable, "-jar", mmt_executable] + args, stderr=subprocess.PIPE if quiet else sys.stderr, stdout=subprocess.PIPE if quiet else sys.stdout, cwd=path) proc.wait() return (proc.returncode == 0) except KeyboardInterrupt: proc.terminate() raise KeyboardInterrupt except FileNotFoundError: err("mmt not found, is it installed?") return False
def install_pack(pack): """Installs a single pack""" (pack, sep, pconfig) = pack.partition(":") pack_setup = get_pack_setup(pack) pack_dir = get_pack_dir(pack) if not pack_setup: return False # Create ext/ if it does not exist. if not os.path.isdir(lmh_locate("ext")): os.mkdir(lmh_locate("ext")) if pack_setup.is_installed(pack_dir): err("Pack", pack, "is already installed, use --update to update. ") return False try: return pack_setup.install(pack_dir, pconfig) except classes.UnsupportedAction: err("Pack", pack, "does not support installing. You may want to --update the pack?") return False
def remove_pack(pack): """Removes a single pack. """ (pack, sep, pconfig) = pack.partition(":") pack_setup = get_pack_setup(pack) pack_dir = get_pack_dir(pack) if not pack_setup: return False if not pack_setup.is_installed(pack_dir): err("Pack", pack, "is not installed, nothing to remove. ") return True if not pack_setup.is_managed(): std("Pack", pack, "is marked as unmanaged, skipping removal. ") return True try: return pack_setup.remove(pack_dir, pconfig) except classes.UnsupportedAction: err("Pack", pack, "does not support removal. Maybe you can remove it manually. ") return False
def do(args, unknown): rep = match_repo(args.repo) if rep == None: err("Unable to find repository '"+args.repo+"' locally. ") return False # Create a generated instance. gen = Generated(rep) if args.branch == None and not args.list and not args.status: err("Branch argument is required unless --list or --status is given. ") return False if args.install: return gen.install_branch(args.branch) if args.init: return gen.init_branch(args.branch) if args.pull: return gen.pull_branch(args.branch) if args.push: return gen.push_branch(args.branch) if args.list: for b in gen.get_all_branches(tuple=False): std(b) return True return gen.print_status()
def status(repos, show_unchanged, remote, *args): """Does git status on all installed repositories """ ret = True for rep in repos: # If we are clean, do nothing if is_clean(rep) and not show_unchanged: continue std("git status", rep) if remote: r_status = get_remote_status(rep) if r_status == "failed": std("Remote status:", term_colors("red")+"Unknown (network issues)", term_colors("normal")) elif r_status == "ok": std("Remote status:", term_colors("green")+"Up-to-date", term_colors("normal")) elif r_status == "pull": std("Remote status:", term_colors("yellow")+"New commits on remote, please pull. ", term_colors("normal")) elif r_status == "push": std("Remote status:", term_colors("yellow")+"New local commits, please push. ", term_colors("normal")) elif r_status == "divergence": std("Remote status:", term_colors("red")+"Remote and local versions have diverged. ", term_colors("normal")) val = git_status(rep, *args) if not val: err("Unable to run git status on", rep) ret = False return ret
def update_pack(pack): """Updates a single pack. """ (pack, sep, pconfig) = pack.partition(":") pack_setup = get_pack_setup(pack) pack_dir = get_pack_dir(pack) if not pack_setup: return False if not pack_setup.is_installed(pack_dir): err("Pack", pack, "is not installed, can not update. ") return False if not pack_setup.is_managed(): std("Pack", pack, "is marked as unmanaged, skipping update. ") return True try: return pack_setup.update(pack_dir, pconfig) except classes.UnsupportedAction: err("Pack", pack, "does not support updating. You may want to --reset the pack. ") return False
def get_metainf_lines(package): """ Gets the lines of the meta-inf file. @param package {string} Package to read meta-inf lines form. @returns {string[]} """ # Find the package root directory. package_dir = find_repo_dir(lmh_locate("content", package)) # Check that repository is installed. if not package_dir: err("Repository", package, "is not installed. Failed to read META-INF. ") return [] # Read the path to meta_inf meta_inf_path = os.path.join(package_dir, "META-INF", "MANIFEST.MF") try: # Try and read the file lines return read_file_lines(meta_inf_path) except: # File is not readable, silently fail. return []
def install(*reps): """Install a repositories and its dependencies""" ret = True reps = list(filter(lambda x:x, [r.strip() for r in reps])) for rep in reps: if not is_installed(rep): std("Starting installation: ", term_colors("blue")+"'"+rep+"'"+term_colors("normal")) (res, deps) = do_install(rep) if not res: err("Failed installation: ", term_colors("red")+"'"+rep+"'"+term_colors("normal")) ret = False else: std("Finished installation: ", term_colors("green")+"'"+rep+"'"+term_colors("normal")) reps.extend([d for d in deps if not d in reps]) else: std("Re-scanning for dependencies: ", term_colors("blue")+"'"+rep+"'"+term_colors("normal")) (res, deps) = do_deps_install(rep) if not res: err("Failed scan: ", term_colors("red")+"'"+rep+"'"+term_colors("normal")) else: std("Finished scan: ", term_colors("green")+"'"+rep+"'"+term_colors("normal")) reps.extend([d for d in deps if not d in reps]) return ret
def push_branch(self, branch): """ Pushes the given branch. """ # Resolve path to branch. (rpath, dpath) = self.get_paths(branch) # make sure it exists if not dpath: err("Unable to find given generated content branch '"+branch+"'. ") return False # and is installed. if not self.is_installed(branch): err("Given generated branch '"+branch+"' is not installed, can not push. ") return False # add all the changes. if not do(dpath, "add", "-A", "."): return False # commit them. if not do(dpath, "commit", "--amend", "--allow-empty", "-m", "Update generated content"): return False # and force push them. if not do(dpath, "push", "--force", "origin", branch): return False # and thats it. return True
def do(args, unknown): if args.reset_all: try: os.remove(config.config_file) return True except: pass return False if args.reset: if args.key == None: err("Missing key. ") return False try: return config.reset_config(args.key) except: pass return False if args.key == None: config.list_config() std() std("Type 'lmh config KEY' to get more information on KEY. ") std("Type 'lmh config KEY VALUE' to change KEY to VALUE. ") return True elif args.value == None: return config.get_config_help(args.key) else: try: return config.set_config(args.key, args.value) except: pass return False
def find_source(name, quiet = False): """ Finds the source of a repository. @param name - Name of repository to find. @param quiet - Should we print output. """ # Check if the result is cached. # In that case we won't have to query again. if name in find_source.cache: return find_source.cache[name] # Iterate over the root urls # and the suffixes. root_urls = get_config("install::sources").rsplit(";") root_suffix = ["", ".git"] for url in root_urls: for url_suf in root_suffix: # Check if the remote repository exists. if exists(url+name+url_suf, False): find_source.cache[name] = url+name+url_suf return url+name+url_suf # We could not find any matching remote. # So send an error message unless we are quiet. if not quiet: err("Can not find remote repository", name) err("Please check install::sources and check your network connection. ") # and we failed. return False
def do_update(self, pack_dir, sstring): """Updates a git controlled package. """ try: return git_pull(pack_dir) except: err("git pull failed to update. Please make sure that you have a network connection. ") err("If you were using a specific version (with the PACKAGE:URL@REFSEPEC syntax), try using --reinstall. ") return False
def rm_untracked(file, t = ""): if not is_tracked(file): try: os.remove(file) std("Removed", t, file) except: err("Unable to remove", file) return False return True
def do(args, unknown): args.source = args.source[0] if not os.path.isfile(args.source) or not args.source.endswith(".tex"): err("Module", args.source, "does not exist or is not a valid module. ") # Remove the .tex args.source = args.source[:-len(".tex")] return create_multi(args.source, args.terms, *args.dest)
def do(args, unknown): repos = match_repo_args(args.repository, args.all) res = calc_deps(repos, apply=args.apply) if res: return True else: err("lmh depcrawl must be run from within a Repository. ") return False
def warn_symbols(fname, syms, symdefs, warns): # fwanr about double things for sym in ["-".join(s[2]) for s in syms]: if sym in symdefs: # Only make a warning if wehaven't done so already warn = (sym, fname) if not (warn in warns): warns.append(warn) err(fname+",", "Symbol", sym+": Found both symdef and symi. ") return True
def warn_symbols(fname, syms, symdefs, warns): # fwanr about double things for sym in ["-".join(s[2]) for s in syms]: if sym in symdefs: # Only make a warning if wehaven't done so already warn = (sym, fname) if not (warn in warns): warns.append(warn) err(fname + ",", "Symbol", sym + ": Found both symdef and symi. ") return True
def log(ordered, *repos): """Prints out log messages on all repositories. """ ret = True def get_log(repo): get_format = lambda frm:git_do_data(repo, "log", "--pretty=format:"+frm+"")[0].split("\n") hash_short = get_format("%h") commit_titles = get_format("%s") dates = get_format("%at") dates_human = get_format("%ad") author_names = get_format("%an") author_mails = get_format("%ae") res = [{ "hash": hash_short[i], "subject": commit_titles[i], "date": int(dates[i]), "date_human": dates_human[i], "author": author_names[i], "author_mail": author_mails[i], "repo": match_repo(repo) } for i in range(len(hash_short))] return res entries = [] for rep in repos: try: entries.extend(get_log(rep)) except Exception as e: err(e) ret = False if ordered: entries.sort(key=lambda e: -e["date"]) strout = "" for entry in entries: strout += "\nRepo: " + entry["repo"] strout += "\nSubject: " + entry["subject"] strout += "\nHash: " + entry["hash"] strout += "\nAuthor: " + entry["author"] + " <" + entry["author_mail"] + ">" strout += "\nDate: " + entry["date_human"] strout += "\n" std_paged(strout, newline=False) return ret
def is_installed(self, branch): """ Checks if a specific branch is installed. """ # Resolve path to branch. (rpath, dpath) = self.get_paths(branch) if not dpath: err("Unable to find given generated content branch. ") return False return os.path.isdir(dpath)
def reset_config(key): """ Resets a given config setting to the default by deleting it from the config file. """ # check if the given key exists if not key in config_meta: err("Option", key, " does not exist, unable to reset. ") return False # load the default and set it to that. set_config(key, str(config_meta[key]["default"]))
def cpanm_make(pack_dir): """Run CPANM make commands for a package. """ _env = perl5env(os.environ) _env.pop("STEXSTYDIR", None) try: call(cpanm_installdeps_args, env=_env, cwd=pack_dir, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr) call(cpanm_installself_args, env=_env, cwd=pack_dir, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr) return True except Exception as e: err(e) err("Failed to run cpanm on", pack_dir) return False
def __init__(self, rep): """ Loads information about generated repositories. """ # Get the repository. self.repo = match_repo(rep) # Check if we exist at all. if not self.repo: err("Unable to find repository '"+rep+"'") # Initialise the cache. self.__branch_cache = None
def resolve_package_spec(packages): """Resolves package specification. """ to_install = set() for p in packages: (pname, sep, pconfig) = p.partition(":") if pname in av_packs["packs"]: to_install.add(p) elif pname in av_packs["groups"]: for q in av_packs["groups"][p]: to_install.add(q+sep+pconfig) else: err("No package or group with the name", pname, "found. ") return sorted(to_install, key=lambda p:av_packs["packs"][p.partition(":")[0]])
def __init__(self, rep): """ Loads information about generated repositories. """ # Get the repository. self.repo = match_repo(rep) # Check if we exist at all. if not self.repo: err("Unable to find repository '" + rep + "'") # Initialise the cache. self.__branch_cache = None
def resolve_package_spec(packages): """Resolves package specification. """ to_install = set() for p in packages: (pname, sep, pconfig) = p.partition(":") if pname in av_packs["packs"]: to_install.add(p) elif pname in av_packs["groups"]: for q in av_packs["groups"][p]: to_install.add(q + sep + pconfig) else: err("No package or group with the name", pname, "found. ") return sorted(to_install, key=lambda p: av_packs["packs"][p.partition(":")[0]])
def export(f=None): """Exports the list of currently installed repositories. """ # Get all locally installed directories installed = match_repos(lmh_locate("content")) if (f == None): for mod in installed: std(mod) return True try: write_file(f, os.linesep.join(installed)) return True except: err("Unable to write %s" % f) return False
def export(f = None): """Exports the list of currently installed repositories. """ # Get all locally installed directories installed = match_repos(lmh_locate("content")) if(f == None): for mod in installed: std(mod) return True try: write_file(f, os.linesep.join(installed)) return True except: err("Unable to write %s" % f) return False
def main(argv=sys.argv[1:]): """Calls the main program with given arguments. """ # Load commands + aliases commands = json.loads(read_file(lmh_locate("lib", "data", "commands.json"))) aliases = json.loads(read_file(lmh_locate("lib", "data", "aliases.json"))) parser = create_parser(submods, commands, aliases) if len(argv) == 0: parser.print_help() return # parse arguments (args, unknown) = parser.parse_known_args(argv) # do the quiet stuff. if args.quiet: lmh.lib.io.__supressStd__ = True lmh.lib.io.__supressErr__ = True if args.non_interactive: lmh.lib.io.__supressIn__ = True # No action. if args.action == None: parser.print_help() return # an alias, so change the arguments. if args.action in aliases: # new argvs argv = shlex.split(aliases[args.action]) + unknown # and re-parse (args, unknown) = parser.parse_known_args(argv) try: if not submods[args.action].allow_unknown and len(unknown) > 0: err("Too many arguments. ") return False except Exception as e: err(e) # run normally. return submods[args.action].do(args, unknown)
def do_install(rep): """ Installs a single repository. """ # pre-installation hook. std("Running pre-installation hook for '"+rep+"' ... ", newline=False) if not hook_pre_install(rep): err("Failed. ") return (False, []) std("Done. ") # Find the remote. std("Finding remote for '"+rep+"' ... ") repoURL = find_source(rep) # Did we find a url? if repoURL == False: std("") err(" Could not find a remote. ") return (False, []) std(" OK, will clone from '"+repoURL+"'") # Clone the repository. if not clone(lmh_locate("content"), repoURL, rep): err("git clone did not exit cleanly, cloning failed. ") err(""" Most likely your network connection is bad. If you are using localmh_docker make sure that you have internet access inside the virtual machine. """) return (False, []) std(" OK. ") # post-installation hook. std("Running post-installation hook for '"+rep+"' ... ", newline=False) if not hook_post_install(rep): err("Failed. ") return (False, []) std("Done. ") # Check for dependencies. return do_deps_install(rep)
def do_install(self, pack_dir, sstring): """Installs a git controlled package. """ (source, branch) = get_item_source(sstring, self.dsource, self.dbranch, self.name) try: # git clone first if not git_clone(lmh_locate("ext"), source, pack_dir): return False # do the checkout if branch != "": return git_do(lmh_locate("ext", pack_dir), "checkout", branch) else: return True except: err("git clone failed to clone", source, ". Check your network connection. ") return False
def load_command(cmd, subparsers): # Load the module module = getattr(getattr(__import__("lmh.commands."+cmd), "commands"), cmd) try: command = module.Command() # Create the sub parser new_parser = subparsers.add_parser(cmd, help=command.help, description=command.help, formatter_class=LMHFormatter,add_help=command.allow_help_arg) # and add some arguments. command.add_parser_args(new_parser) except Exception as e: err(e) err("Command", cmd, "failed to load. ") command = None # return the new command instance. return command
def get_config_help(key): """ Prints help to stdout about the given configuration settting. """ # check if the key exists if not key in config_meta: err("Option", key, " does not exist. ") return False # get its meta-information meta = config_meta[key] # and print that std(format_type(meta["type"]), key) std(meta["help"]) std("Current Value: " + json.dumps(get_config(key))) std("Default Value: " + json.dumps(meta["default"])) return True
def install_branch(self, branch): """ Installs the given branch. """ # Resolve path to branch. (rpath, dpath) = self.get_paths(branch) # make sure it exists if not dpath: err("Unable to find given generated content branch '" + branch + "'. ") return False # and is not installed. if self.is_installed(branch): err("Given generated branch '" + branch + "' already installed. Did you want to pull?") return False (o, e) = do_data(rpath, "config", "--local", "--get", "remote.origin.url") if not do(rpath, "rev-parse", "--verify", "--quiet", branch): if not do(rpath, "branch", branch, "--track", "origin/" + branch): return False # Clone it shared if not do(rpath, "clone", rpath, dpath, "--shared", "-b", branch): return False # set up .git/objects/info/alternates relatively if not write_file(os.path.join(dpath, ".git/objects/info/alternates"), "../../../.git/objects"): return False # and set the origin correctly. if not do(dpath, "remote", "set-url", "origin", o.rstrip("\n")): return False return do(rpath, "branch", "-D", branch)
def reset_pack(pack): """Resets a single pack. """ (pack, sep, pconfig) = pack.partition(":") pack_setup = get_pack_setup(pack) pack_dir = get_pack_dir(pack) if not pack_setup: return False if not pack_setup.is_managed(): std("Pack", pack, "is marked as unmanaged, skipping reset. ") return True if not pack_setup.is_installed(pack_dir): err("Pack", pack, "is not installed, skipping removal. ") else: if not remove_pack(pack+sep+pconfig): err("Pack", pack, "could not be removed, --reset has failed. ") return False if not install_pack(pack+sep+pconfig): err("Pack", pack, "could not be (re)installed, --reset has failed. ") return False return True
def reset_pack(pack): """Resets a single pack. """ (pack, sep, pconfig) = pack.partition(":") pack_setup = get_pack_setup(pack) pack_dir = get_pack_dir(pack) if not pack_setup: return False if not pack_setup.is_managed(): std("Pack", pack, "is marked as unmanaged, skipping reset. ") return True if not pack_setup.is_installed(pack_dir): err("Pack", pack, "is not installed, skipping removal. ") else: if not remove_pack(pack + sep + pconfig): err("Pack", pack, "could not be removed, --reset has failed. ") return False if not install_pack(pack + sep + pconfig): err("Pack", pack, "could not be (re)installed, --reset has failed. ") return False return True
def do_pull(rep, needs_update): """ Actually pulls a repository. """ # pre-installation hook. std("Running pre-update hook for '" + rep + "' ... ", newline=False) if not hook_pre_pull(rep): err("Failed. ") return (False, []) std("Done. ") ret = True if needs_update: std("Running git pull ...") rgp = git_pull(match_repo(rep, abs=True)) ret = rgp and ret if rgp: std("OK") else: err("Failed (merge conflicts or network issues?)") ret = False else: std("Nothing to pull ...") std("Running post-update hook for '" + rep + "' ... ", newline=False) if not hook_post_pull(rep): err("Failed. ") return (False, []) std("Done. ") return ret
def pull_branch(self, branch): """ Pulls the given branch. """ # Resolve path to branch. (rpath, dpath) = self.get_paths(branch) # make sure it exists if not dpath: err("Unable to find given generated content branch '"+branch+"'. ") return False # make sure it exists if not dpath: err("Unable to find given generated content branch '"+branch+"'. ") return False # and is installed. if not self.is_installed(branch): err("Given generated branch '"+branch+"' is not installed, can not pull. ") return False # Fetch origin in the clone repo. if not do(rpath, "fetch", "--depth", "1", "origin", branch): return False # Hard reset this repository. if not do(dpath, "reset", "--hard", "origin/"+branch): return False # Run some housekeeping in the parent repo return do(rpath, "gc", "--auto")
def do_pull(rep, needs_update): """ Actually pulls a repository. """ # pre-installation hook. std("Running pre-update hook for '"+rep+"' ... ", newline=False) if not hook_pre_pull(rep): err("Failed. ") return (False, []) std("Done. ") ret = True if needs_update: std("Running git pull ...") rgp = git_pull(match_repo(rep, abs=True)) ret = rgp and ret if rgp: std("OK") else: err("Failed (merge conflicts or network issues?)") ret = False else: std("Nothing to pull ...") std("Running post-update hook for '"+rep+"' ... ", newline=False) if not hook_post_pull(rep): err("Failed. ") return (False, []) std("Done. ") return ret
def do(args, unknown): if not args.no_check and not check_deps(): err("Dependency check failed. ") err("Cannot perform specefied action. ") err("Use --no-check to skip checking dependencies. ") return False if len(args.pack) == 0: args.pack = ["default"] if args.saction == "update": # Update self as well when calling lmh setup --update args.pack += ["self"] if args.saction == "install": return lmh.lib.packs.install(*args.pack) elif args.saction == "update": return lmh.lib.packs.update(*args.pack) elif args.saction == "remove": return lmh.lib.packs.remove(*args.pack) elif args.saction == "reset": return lmh.lib.packs.reset(*args.pack) elif args.saction == "manage": return lmh.lib.packs.manage(*args.pack) elif args.saction == "unmanage": return lmh.lib.packs.unmanage(*args.pack) elif args.saction == "status": return lmh.lib.packs.status(*args.pack) else: std("No setup action specefied, assuming --install. ") std("Please specify some action in the future. ") return lmh.lib.packs.install(*args.pack)
def find_cached(files, match, replace = None, replace_match = None): """Finds and replaces inside of files. """ # Make sure match and replace are arrays match = [match] if is_string(match) else match if replace != None: replace = [replace] if is_string(replace) else replace if len(replace) != len(match): err("Find and Replace patterns are not of the same length. ") return False rep = False for file in files: repo = os.path.relpath(find_repo_dir(file), lmh_locate("content")) matcher = [Template(m).substitute(repo=repo) for m in match] if replace != None: rep = find_and_replace_file(file, matcher, replace, replace_match = replace_match) or rep else: rep = find_file(file, matcher) or rep return rep
def find_sds(fname, warns=[]): # skip non-language bindings languageFilePattern = r"\.(\w+)\.tex$" # Find the associated module fmodname = re.sub(languageFilePattern, ".tex", fname) content = read_file(fname) if len(re.findall(languageFilePattern, fname)) == 0: # This is what we have syms = find_all_symis(content) symdefs = find_all_symdefs(content) # for non-language-bndings return [warn_symbols(fname, syms, symdefs, warns)] # Try and read the other file try: modcontent = read_file(fmodname) except IOError: err("Missing module:", fmodname) return [False] # FInd all the symbolds and definitions defs = find_all_defis(content) # This is what we have syms = find_all_symis(modcontent) symdefs = find_all_symdefs(modcontent) # Warn about symbols warn_symbols(fmodname, syms, symdefs, warns) if defs == None: defs = [] if syms == None: syms = [] return (fmodname, defs, syms, symdefs, modcontent)
def do(args, unknown): # If there are no repositories, check everything for dependencies. if len(args.spec) == 0: std("Nothing to install, re-installing all existing repositories. ") return install(*match_repos(lmh_locate("content"))) if not get_config("install::noglobs"): args.spec = ls_remote(*args.spec) if len(args.spec) == 0: err("Nothing to install...") return True if args.no_confirm_install: std("Picked", len(args.spec),"repositories. ") else: std("Picked", len(args.spec),"repositories: ") std(*args.spec) if read_raw("Continue (y/N)?").lower() != "y": err("Installation aborted. ") return False return install(*args.spec)
def get_config(key): """ Gets a given configuration setting. If it does not exist in the configuration file, the default will be returned """ # check if the given key exists in the configuration if not key in config_meta: err("Option", key, "does not exist. ") raise KeyError # Read the setting from the config file # TODO: Think about verifying the type try: data = json.loads(read_file(config_file)) return data[key] except: pass # return the default value return config_meta[key]["default"]