def multi_burn( self, names=None, devices=None, verbose=False, password=None, ): """ Given multiple names, burn them """ if devices is None: Console.error('Device not specified.') return if names is None: Console.error('Names to burn not specified') return names = Parameter.expand(names) devices = Parameter.expand(devices) if len(devices) > 1: Console.error('We do not yet support burning on multiple devices') return for name in names: self.burn(name=name, device=devices[0], verbose=verbose, password=password) Console.ok('Finished burning all cards')
def names(self, collection=None, cloud=None, kind=None, regex=None): """ finds all names in the specified collections. The parameters, collection, cloud, and kind can all be Parameters that get expanded to lists. All names from all collections are merged into the result. With kwargs a search query on the names could be added. Example: cm = CmDatabase() for kind in ['vm', "image", "flavor"]: names = cm.names(cloud="chameleon", kind=kind) print (names) names = cm.names(cloud="chameleon,azure", kind="vm") names = cm.names(collection="chameleon-image", regex="^CC-") names = cm.names(collection="chameleon-image", regex=".*Ubuntu.*") :param collection: The collections :param cloud: The clouds :param kind: The kinds :param regex: A query applied to name :return: """ MongoDBController().start_if_not_running() collections = Parameter.expand(collection) clouds = Parameter.expand(cloud) kinds = Parameter.expand(kind) result = [] def add(collection): col = self.db[collection] if not regex: entries = col.find({}, {"name": 1, "_id": 0}) else: entries = col.find({"name": { '$regex': regex }}, { "name": 1, "_id": 0 }) for entry in entries: result.append(entry['name']) # # if collection is none but kind is not none find all collections # matching THIS IS NOT YET IMPLEMENTED # if kinds and clouds: for _kind in kinds: for _cloud in clouds: _collection = f"{_cloud}-{_kind}" add(_collection) if collections: for _collection in collections: add(_collection) return result
def execute(self, arguments): """ pi spark check [--master=MASTER] [--workers=WORKERS] pi spark setup [--master=MASTER] [--workers=WORKERS] pi spark start --master=MASTER pi spark stop --master=MASTER pi spark test --master=MASTER pi spark uninstall --master=MASTER [--workers=WORKERS] :param arguments: :return: """ self.master = arguments.master self.workers = Parameter.expand(arguments.workers) master = [] hosts = [] workers_only = [] if arguments.master: hosts.append(arguments.master) master = arguments.master if arguments.workers: hosts = hosts + Parameter.expand(arguments.workers) workers_only = Parameter.expand(arguments.workers) if arguments.dryrun: self.dryrun = True if hosts is None: Console.error("You need to specify at least one master or worker") return "" if arguments.setup: self.setup(master, workers_only) elif arguments.start: self.run_script(name="spark.start", hosts=master) elif arguments.stop: self.run_script(name="spark.stop", hosts=master) elif arguments.test: self.test(master) #self.run_script(name="spark.test", hosts=self.master) elif arguments.check: self.run_script(name="spark.check", hosts=hosts) elif arguments.uninstall: self.uninstall(master, workers_only)
def hostnames(host): if host == "volta": names = Parameter.expand("r-00[5-6]") else: names = Parameter.expand("r-00[3-4]") max_gpus = 8 # this is for now hard coded valid = [] for name in names: if name not in reserved and status[name] + gpus <= max_gpus: valid.append(name) return valid
def do_mongo(self, args, arguments): """ :: Usage: mongo deploy --config FILE mongo deploy --ips=IPS --names=NAMES --shards=SHARDS --replicas=REPLICAS This command does some useful things. Arguments: FILE a file name Options: -f specify the file Description: > mongo deploy --config FILE > mongo deploy --ips=10.0.0.[1-5] > --names=master,worker[2-5] > --shards=3 > --replicas=2 """ map_parameters(arguments, "config", "ips", "names", "replicas") if arguments.deploy and arguments.config: print(f"configure from file {arguments.FILE}") config = parse_config(arguments.FILE) valid = validate_config(config) if arguments.deploy and arguments.ips: names = Parameter.expand(arguments.names) ips = Parameter.expand(arguments.ips) if len(ips) != len(names): Console.error("The length of ips and names do not match") print(f"configure") print(ips) print(names) print(arguments.shards) print(arguments.replicas) else: Console.error("parameters not specified correctly") return ""
def add_group(self, name=None, group=None): names = Parameter.expand(name) groups = Parameter.expand(group) cloud = "local" db = CmDatabase() keys = db.find(collection=f"{cloud}-key") for key in keys: if key["name"] in names: for _group in groups: if _group not in key["group"]: key["group"].append(_group) return keys
def execute(self, arguments): """ pi spark worker add [--master=MASTER] [--workers=WORKERS] [--dryrun] pi spark worker remove [--master=MASTER] [--workers=WORKERS] [--dryrun] pi spark setup [--master=MASTER] [--workers=WORKERS] [--dryrun] pi spark start [--master=MASTER] [--workers=WORKERS] [--dryrun] pi spark stop [--master=MASTER] [--workers=WORKERS] [--dryrun] pi spark test [--master=MASTER] [--workers=WORKERS] [--dryrun] pi spark check [--master=MASTER] [--workers=WORKERS] [--dryrun] :param arguments: :return: """ self.master = arguments.master self.workers = Parameter.expand(arguments.workers) hosts = [] if arguments.master: hosts.append(arguments.master) master = arguments.master if arguments.workers: hosts = hosts + Parameter.expand(arguments.workers) master = [] if arguments.dryrun: self.dryrun = True if hosts is None: Console.error("You need to specify at least one master or worker") return "" if arguments.setup: self.setup(master,hosts) elif arguments.start: self.run_script(name="spark.start", hosts=hosts) elif arguments.stop: self.run_script(name="spark.stop", hosts=hosts) elif arguments.test: #self.test(hosts) self.run_script(name="spark.test", hosts=self.master) elif arguments.check: self.run_script(name="spark.check", hosts=hosts)
def ssh_keygen(hosts=None, filename="~/.ssh/id_rsa", username=None, processors=3, dryrun=False, verbose=True): """ generates the keys on the specified hosts. this fonction does not work well as it still will aski if we overwrite. :param hosts: :param filename: :param username: :param output: :param dryrun: :param verbose: :return: """ hosts = Parameter.expand(hosts) command = f'ssh-keygen -q -N "" -f {filename} <<< y' result_keys = Host.ssh(hosts=hosts, command=command, username=username, dryrun=dryrun, processors=processors, executor=os.system) result_keys = Host.ssh(hosts=hosts, processors=processors, command='cat .ssh/id_rsa.pub', username=username) return result_keys
def put(hosts=None, source=None, destination=None, username=None, key="~/.ssh/id_rsa.pub", shell=False, processors=3, dryrun=False, verbose=False): """ :param command: the command to be executed :param hosts: a list of hosts to be checked :param username: the usernames for the hosts :param key: the key for logging in :param processors: the number of parallel checks :return: list of dicts representing the ping result """ hosts = Parameter.expand(hosts) key = path_expand(key) command = [ 'scp', "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", '-i', key, source, "{host}:{destination}" ] result = Host.run(hosts=hosts, command=command, destination=destination, shell=False) return result
def check(hosts=None, username=None, key="~/.ssh/id_rsa.pub", processors=3): # # BUG: this code has a bug and does not deal with different # usernames on the host to be checked. # """ :param hosts: a list of hosts to be checked :param username: the usernames for the hosts :param key: the key for logging in :param processors: the number of parallel checks :return: list of dicts representing the ping result """ hosts = Parameter.expand(hosts) result = Host.ssh(hosts=hosts, command='hostname', username=username, key=key, processors=processors) return result
def add(self, groupname=None, keyname=None): """ adds a key to a given group. If the group does not exist, it will be created. :param groupname: :param keyname: :return: """ # print('In KeyGroup') # print('name: ' , keyname) new_key = keyname if type(keyname) == str: new_key = Parameter.expand(keyname) elif type(keyname) == list: pass else: raise ValueError("key have wrong type") # noinspection PyUnusedLocal try: entry = self.find(name=groupname)[0] except Exception as e: entry = {'name': groupname, 'keys': [keyname]} if groupname is not None: entry['keys'] += list(set(new_key)) entry['keys'] = list(set(entry['keys'])) else: entry['keys'] = list(set(new_key)) return self.update_dict_list([entry])
def setup(self, master=None, hosts=None): # Setup master if master is None and hosts: Console.error("You must specify a master to set up nodes") raise ValueError # Setup Spark on the master if master is not None: if type(master) != list: master = Parameter.expand(master) # # TODO - bug I should be able to run this even if I am not on master # banner(f"Setup Master: {master[0]}") self.run_script(name="spark.setup", hosts=master) update_bashrc() # Setup workers and update master's slaves file # # if hosts is not None: if master is not None: banner(f"Get files from {master[0]}") create_spark_setup_worker(self) create_spark_bashrc_txt(self) self.run_script(name="copy.spark.to.worker", hosts=hosts) update_slaves(self) # Print created cluster self.view()
def add(self, **kwargs): if "host" not in kwargs: Console.error("no id specified") sys.exit(1) hosts = hostlist.expand_hostlist(kwargs['host']) if 'ip' in kwargs: ips = Parameter.expand(kwargs['ip']) else: ips = [None for i in hosts] if ips is None: ips = [None for i in hosts] for host, ip in zip(hosts, ips): if host in self.data: entry = self.data[host] else: entry = dict(self.entry) self.data[host] = entry for key, value in kwargs.items(): entry[key] = value entry['ip'] = ip entry['host'] = host for attribute in entry: self.data[host][attribute] = entry[attribute]
def gather(user, names, source, destination="~/.ssh/authorized_keys", dryrun=False, append=False, tmp=None): if type(names) != list: _names = Parameter.expand(names) tmp_dir = tmp or path_expand("~/.cloudmesh/tmp") Shell.mkdir(tmp_dir) destinations = [] for name in _names: destinations.append("{name}:{file}") directory = os.path.dirname(destination) Shell.mkdir(directory) if not append: Shell.rm(path_expand(destination)) writefile(destination, "") for name in _names: source = f"{user}@{name}:{destination}" print(f"{source} -> {tmp_dir}/{destination}") result = Host.scp(source, f"{tmp_dir}/{destination}-{name}", dryrun) with open(path_expand(destination), 'a') as file: for filename in glob(tmp_dir): content = readfile(filename) file.write(content)
def setup(self): print() self.user = Config()["cloudmesh"]["profile"]["user"] self.clouduser = '******' self.name_generator = Name(experiment="exp", group="grp", user=self.user, kind="vm", counter=1) self.name = str(self.name_generator) self.name_generator.incr() self.new_name = str(self.name_generator) variables = Variables() clouds = Parameter.expand(variables['cloud']) cloud = clouds[0] self.p = Provider(name=cloud) self.secgroupname = "CM4TestSecGroup" self.secgrouprule = { "ip_protocol": "tcp", "from_port": 8080, "to_port": 8088, "ip_range": "129.79.0.0/16" } self.testnode = None print("\n")
def run(self, script=None, hosts=None, username=None, processors=4, verbose=False): results = [] if type(hosts) != list: hosts = Parameter.expand(hosts) for command in script.splitlines(): print(hosts, "->", command) if command.startswith("#") or command.strip() == "": pass # print (command) elif len(hosts) == 1 and hosts[0] == self.hostname: os.system(command) elif len(hosts) == 1 and hosts[0] != self.hostname: host = hosts[0] os.system(f"ssh {host} {command}") else: result = Host.ssh(hosts=hosts, command=command, username=username, key="~/.ssh/id_rsa.pub", processors=processors, executor=os.system) results.append(result) if verbose: pprint(results) for result in results: print(Printer.write(result, order=['host', 'stdout'])) return results
def sequence_remote( led=None, hosts=None, username=None, rate=None, processors=3): if led not in [1, 0]: raise ValueError("Led number is wrong") rate = float(rate or 0.5) hosts = Parameter.expand(hosts) for host in hosts: LED.set_remote( led=led, value="0", hosts=host, username=username, processors=processors) time.sleep(rate) LED.set_remote( led=led, value="1", hosts=host, username=username, processors=processors) time.sleep(rate) return None
def main(): arguments = docopt(__doc__) tags = Parameter.expand(arguments["VERSIONS"]) found = Shell.run("git tag").strip().splitlines() # print (found) for tag in tags: if tag in found: print(f"Removing tag {tag}") script = [ f"git tag -d {tag}", f"git push origin :refs/tags/{tag}" ] if arguments["--dryrun"]: print(" " + '\n '.join(script)) else: try: for line in script: os.system(line) Console.ok(f"{tag} deleted") except: Console.error ("Deletion failed") else: Console.error(f"{tag} does not exist")
def add(self, name=None): """ adds a key to a given group. If the group does not exist, it will be created. :param name: :param keys: :return: """ new_key = name if type(name) == str: new_key = Parameter.expand(name) elif type(name) == list: pass else: raise ValueError("key have wrong type") # noinspection PyUnusedLocal try: entry = self.find(name=name)[0] except Exception as e: entry = { 'name': name } if name is not None: old = list(entry['name']) entry['name'] = list(set(new_key + old)) return self.update_dict_list([entry])
def get_filename(filename, hosts): if filename is not None: return filename if type(hosts) == str: hosts = Parameter.expand(hosts) label = hosts[0] return path_expand(f"~/.ssh/cluster_keys_{label}")
def setup(self): variables = Variables() print(variables['storage']) self.service = Parameter.expand(variables['storage'])[0] self.p = Provider(service=self.service) self.sourcedir = path_expand("~/.cloudmesh/storage/test") print()
def get_names(arguments, variables): names = arguments["NAME"] or arguments["NAMES"] or arguments[ "--name"] or variables["vm"] if names is None: return None else: return Parameter.expand(names)
def setup(self): variables = Variables() self.service = Parameter.expand(variables['storage'])[0] self.p = Provider(service=self.service) self.sourcedir = os.path.expanduser( "~/Documents/cloudmesh/storage/test") print()
def delete(self, name=None): """ deletes the groups :param name: :param rules: :return: """ delete_key = name if type(name) == str: delete_key = Parameter.expand(name) elif type(name) == list: pass else: raise ValueError("key have wrong type") delete_key = set(delete_key) entry = self.find(name=name)[0] if name is not None: old = set(entry['key']) old -= delete_key entry['key'] = list(old) return entry
def add(self, name=None, services=None, category=None): # check if non and raise error if type(services) == str: services = Parameter.expand(services) # cm = CmDatabase() entry = { 'cm': { "name": name, "cloud": self.cloud, "kind": self.kind } } entry['members'] = [] # find in db old = DictList(entry['members']) entries = [{'name': service, 'kind': category} for service in services] for entry in old: if entry not in entries: entries.append(old[entry]) entry['members'] = entries return [entry]
def delete(self, name=None, rules=None): """ deletes the groups :param name: :param rules: :return: """ delete_rules = rules if type(rules) == str: delete_rules = Parameter.expand(rules) elif type(rules) == list: pass else: raise ValueError("rules have wrong type") delete_rules = set(delete_rules) entry = self.find(name=name)[0] if rules is not None: old = set(entry['rules']) old -= delete_rules entry['rules'] = list(old) return entry
def collections(self, name=None, regex=None): """ the names of all collections :param name: if set, only look at these collections instead of all collections :param regex: a regular expression on the names of the collections :return: list of names of all collections Example: collections = cm.collections(regex=".*-vm") """ MongoDBController().start_if_not_running() names = None if name: if type(name) == list: names = name else: names = Parameter.expand(name) else: names = self.db.collection_names() if regex: r = re.compile(regex) _names = list(filter(r.match, names)) names = _names return names
def suspend(self, names=None): """ NOT YET IMPLEMENTED. suspends the node with the given name. :param name: the name of the node :return: The dict representing the node """ HEADING(c=".") names = Parameter.expand(names) nodes = self.list(raw=True) for node in nodes: if node.name in names: r = self.cloudman.ex_stop_node(self._get_node(node.name), deallocate=False) print(r) self.cloudman.destroy_node(node) raise NotImplementedError # # should return the updated names dict, e.g. status and so on # return None
def __init__(self, hostname=None, ip=None, dryrun=False, no_diagram=False): self.dryrun = dryrun or False self.hostnames_str = hostname self.ips_str = ip self.hostnames = hostnames = hostname or "red,red[01-02]" self.ips = ips = ip or "10.0.0.[1-3]" self.ssid = get_ssid() self.imaged = "" self.wifipassword = "" self.no_diagram = no_diagram hostnames = Parameter.expand(hostnames) manager, workers = Host.get_hostnames(hostnames) if workers is None: n = 1 else: n = len(workers) + 1 if ip is None: ips = Parameter.expand(f"10.1.1.[1-{n}]") else: ips = Parameter.expand(ips) # cluster_hosts = tuple(zip(ips, hostnames)) self.key = path_expand("~/.ssh/id_rsa.pub") banner("Parameters", figlet=True) print("Manager: ", manager) print("Workers: ", workers) print("IPS: ", ips) print("Key: ", self.key) print("Dryrun: ", self.dryrun) self.manager = manager self.workers = workers self.ips = ips self.load_data() if not self.no_diagram: self.create_diag(self.manager) self.create_layout() # sg.change_look_and_feel('SystemDefault') self.window = sg.Window('Cloudmesh Pi Burn', self.layout, resizable=True, size=window_size)
def get_names(arguments, variables): names = arguments["NAME"] or arguments["NAMES"] or arguments[ "--name"] or variables["vm"] if names is None: Console.error("you need to specify a vm") return None else: return Parameter.expand(names)