def multi_burn(
self,
names=None,
devices=None,
verbose=False,
password=None,
):
"""
Given multiple names, burn them
"""
if devices is None:
Console.error('Device not specified.')
return
if names is None:
Console.error('Names to burn not specified')
return
names = Parameter.expand(names)
devices = Parameter.expand(devices)
if len(devices) > 1:
Console.error('We do not yet support burning on multiple devices')
return
for name in names:
self.burn(name=name,
device=devices[0],
verbose=verbose,
password=password)
Console.ok('Finished burning all cards')
def names(self, collection=None, cloud=None, kind=None, regex=None):
"""
finds all names in the specified collections. The parameters,
collection, cloud, and kind can all be Parameters that get expanded
to lists. All names from all collections are merged into the result.
With kwargs a search query on the names could be added.
Example:
cm = CmDatabase()
for kind in ['vm', "image", "flavor"]:
names = cm.names(cloud="chameleon", kind=kind)
print (names)
names = cm.names(cloud="chameleon,azure", kind="vm")
names = cm.names(collection="chameleon-image", regex="^CC-")
names = cm.names(collection="chameleon-image", regex=".*Ubuntu.*")
:param collection: The collections
:param cloud: The clouds
:param kind: The kinds
:param regex: A query applied to name
:return:
"""
MongoDBController().start_if_not_running()
collections = Parameter.expand(collection)
clouds = Parameter.expand(cloud)
kinds = Parameter.expand(kind)
result = []
def add(collection):
col = self.db[collection]
if not regex:
entries = col.find({}, {"name": 1, "_id": 0})
else:
entries = col.find({"name": {
'$regex': regex
}}, {
"name": 1,
"_id": 0
})
for entry in entries:
result.append(entry['name'])
#
# if collection is none but kind is not none find all collections
# matching THIS IS NOT YET IMPLEMENTED
#
if kinds and clouds:
for _kind in kinds:
for _cloud in clouds:
_collection = f"{_cloud}-{_kind}"
add(_collection)
if collections:
for _collection in collections:
add(_collection)
return result
def execute(self, arguments):
"""
pi spark check [--master=MASTER] [--workers=WORKERS]
pi spark setup [--master=MASTER] [--workers=WORKERS]
pi spark start --master=MASTER
pi spark stop --master=MASTER
pi spark test --master=MASTER
pi spark uninstall --master=MASTER [--workers=WORKERS]
:param arguments:
:return:
"""
self.master = arguments.master
self.workers = Parameter.expand(arguments.workers)
master = []
hosts = []
workers_only = []
if arguments.master:
hosts.append(arguments.master)
master = arguments.master
if arguments.workers:
hosts = hosts + Parameter.expand(arguments.workers)
workers_only = Parameter.expand(arguments.workers)
if arguments.dryrun:
self.dryrun = True
if hosts is None:
Console.error("You need to specify at least one master or worker")
return ""
if arguments.setup:
self.setup(master, workers_only)
elif arguments.start:
self.run_script(name="spark.start", hosts=master)
elif arguments.stop:
self.run_script(name="spark.stop", hosts=master)
elif arguments.test:
self.test(master)
#self.run_script(name="spark.test", hosts=self.master)
elif arguments.check:
self.run_script(name="spark.check", hosts=hosts)
elif arguments.uninstall:
self.uninstall(master, workers_only)
def hostnames(host):
if host == "volta":
names = Parameter.expand("r-00[5-6]")
else:
names = Parameter.expand("r-00[3-4]")
max_gpus = 8 # this is for now hard coded
valid = []
for name in names:
if name not in reserved and status[name] + gpus <= max_gpus:
valid.append(name)
return valid
def do_mongo(self, args, arguments):
"""
::
Usage:
mongo deploy --config FILE
mongo deploy --ips=IPS
--names=NAMES
--shards=SHARDS
--replicas=REPLICAS
This command does some useful things.
Arguments:
FILE a file name
Options:
-f specify the file
Description:
> mongo deploy --config FILE
> mongo deploy --ips=10.0.0.[1-5]
> --names=master,worker[2-5]
> --shards=3
> --replicas=2
"""
map_parameters(arguments, "config", "ips", "names", "replicas")
if arguments.deploy and arguments.config:
print(f"configure from file {arguments.FILE}")
config = parse_config(arguments.FILE)
valid = validate_config(config)
if arguments.deploy and arguments.ips:
names = Parameter.expand(arguments.names)
ips = Parameter.expand(arguments.ips)
if len(ips) != len(names):
Console.error("The length of ips and names do not match")
print(f"configure")
print(ips)
print(names)
print(arguments.shards)
print(arguments.replicas)
else:
Console.error("parameters not specified correctly")
return ""
def add_group(self, name=None, group=None):
names = Parameter.expand(name)
groups = Parameter.expand(group)
cloud = "local"
db = CmDatabase()
keys = db.find(collection=f"{cloud}-key")
for key in keys:
if key["name"] in names:
for _group in groups:
if _group not in key["group"]:
key["group"].append(_group)
return keys
def execute(self, arguments):
"""
pi spark worker add [--master=MASTER] [--workers=WORKERS] [--dryrun]
pi spark worker remove [--master=MASTER] [--workers=WORKERS] [--dryrun]
pi spark setup [--master=MASTER] [--workers=WORKERS] [--dryrun]
pi spark start [--master=MASTER] [--workers=WORKERS] [--dryrun]
pi spark stop [--master=MASTER] [--workers=WORKERS] [--dryrun]
pi spark test [--master=MASTER] [--workers=WORKERS] [--dryrun]
pi spark check [--master=MASTER] [--workers=WORKERS] [--dryrun]
:param arguments:
:return:
"""
self.master = arguments.master
self.workers = Parameter.expand(arguments.workers)
hosts = []
if arguments.master:
hosts.append(arguments.master)
master = arguments.master
if arguments.workers:
hosts = hosts + Parameter.expand(arguments.workers)
master = []
if arguments.dryrun:
self.dryrun = True
if hosts is None:
Console.error("You need to specify at least one master or worker")
return ""
if arguments.setup:
self.setup(master,hosts)
elif arguments.start:
self.run_script(name="spark.start", hosts=hosts)
elif arguments.stop:
self.run_script(name="spark.stop", hosts=hosts)
elif arguments.test:
#self.test(hosts)
self.run_script(name="spark.test", hosts=self.master)
elif arguments.check:
self.run_script(name="spark.check", hosts=hosts)
def ssh_keygen(hosts=None,
filename="~/.ssh/id_rsa",
username=None,
processors=3,
dryrun=False,
verbose=True):
"""
generates the keys on the specified hosts.
this fonction does not work well as it still will aski if we overwrite.
:param hosts:
:param filename:
:param username:
:param output:
:param dryrun:
:param verbose:
:return:
"""
hosts = Parameter.expand(hosts)
command = f'ssh-keygen -q -N "" -f {filename} <<< y'
result_keys = Host.ssh(hosts=hosts,
command=command,
username=username,
dryrun=dryrun,
processors=processors,
executor=os.system)
result_keys = Host.ssh(hosts=hosts,
processors=processors,
command='cat .ssh/id_rsa.pub',
username=username)
return result_keys
def put(hosts=None,
source=None,
destination=None,
username=None,
key="~/.ssh/id_rsa.pub",
shell=False,
processors=3,
dryrun=False,
verbose=False):
"""
:param command: the command to be executed
:param hosts: a list of hosts to be checked
:param username: the usernames for the hosts
:param key: the key for logging in
:param processors: the number of parallel checks
:return: list of dicts representing the ping result
"""
hosts = Parameter.expand(hosts)
key = path_expand(key)
command = [
'scp', "-o", "StrictHostKeyChecking=no", "-o",
"UserKnownHostsFile=/dev/null", '-i', key, source,
"{host}:{destination}"
]
result = Host.run(hosts=hosts,
command=command,
destination=destination,
shell=False)
return result
def check(hosts=None,
username=None,
key="~/.ssh/id_rsa.pub",
processors=3):
#
# BUG: this code has a bug and does not deal with different
# usernames on the host to be checked.
#
"""
:param hosts: a list of hosts to be checked
:param username: the usernames for the hosts
:param key: the key for logging in
:param processors: the number of parallel checks
:return: list of dicts representing the ping result
"""
hosts = Parameter.expand(hosts)
result = Host.ssh(hosts=hosts,
command='hostname',
username=username,
key=key,
processors=processors)
return result
def add(self, groupname=None, keyname=None):
"""
adds a key to a given group. If the group does not exist, it will be
created.
:param groupname:
:param keyname:
:return:
"""
# print('In KeyGroup')
# print('name: ' , keyname)
new_key = keyname
if type(keyname) == str:
new_key = Parameter.expand(keyname)
elif type(keyname) == list:
pass
else:
raise ValueError("key have wrong type")
# noinspection PyUnusedLocal
try:
entry = self.find(name=groupname)[0]
except Exception as e:
entry = {'name': groupname, 'keys': [keyname]}
if groupname is not None:
entry['keys'] += list(set(new_key))
entry['keys'] = list(set(entry['keys']))
else:
entry['keys'] = list(set(new_key))
return self.update_dict_list([entry])
def setup(self, master=None, hosts=None):
# Setup master
if master is None and hosts:
Console.error("You must specify a master to set up nodes")
raise ValueError
# Setup Spark on the master
if master is not None:
if type(master) != list:
master = Parameter.expand(master)
#
# TODO - bug I should be able to run this even if I am not on master
#
banner(f"Setup Master: {master[0]}")
self.run_script(name="spark.setup", hosts=master)
update_bashrc()
# Setup workers and update master's slaves file
#
#
if hosts is not None:
if master is not None:
banner(f"Get files from {master[0]}")
create_spark_setup_worker(self)
create_spark_bashrc_txt(self)
self.run_script(name="copy.spark.to.worker", hosts=hosts)
update_slaves(self)
# Print created cluster
self.view()
def add(self, **kwargs):
if "host" not in kwargs:
Console.error("no id specified")
sys.exit(1)
hosts = hostlist.expand_hostlist(kwargs['host'])
if 'ip' in kwargs:
ips = Parameter.expand(kwargs['ip'])
else:
ips = [None for i in hosts]
if ips is None:
ips = [None for i in hosts]
for host, ip in zip(hosts, ips):
if host in self.data:
entry = self.data[host]
else:
entry = dict(self.entry)
self.data[host] = entry
for key, value in kwargs.items():
entry[key] = value
entry['ip'] = ip
entry['host'] = host
for attribute in entry:
self.data[host][attribute] = entry[attribute]
def gather(user,
names,
source,
destination="~/.ssh/authorized_keys",
dryrun=False,
append=False,
tmp=None):
if type(names) != list:
_names = Parameter.expand(names)
tmp_dir = tmp or path_expand("~/.cloudmesh/tmp")
Shell.mkdir(tmp_dir)
destinations = []
for name in _names:
destinations.append("{name}:{file}")
directory = os.path.dirname(destination)
Shell.mkdir(directory)
if not append:
Shell.rm(path_expand(destination))
writefile(destination, "")
for name in _names:
source = f"{user}@{name}:{destination}"
print(f"{source} -> {tmp_dir}/{destination}")
result = Host.scp(source, f"{tmp_dir}/{destination}-{name}",
dryrun)
with open(path_expand(destination), 'a') as file:
for filename in glob(tmp_dir):
content = readfile(filename)
file.write(content)
def setup(self):
print()
self.user = Config()["cloudmesh"]["profile"]["user"]
self.clouduser = '******'
self.name_generator = Name(experiment="exp",
group="grp",
user=self.user,
kind="vm",
counter=1)
self.name = str(self.name_generator)
self.name_generator.incr()
self.new_name = str(self.name_generator)
variables = Variables()
clouds = Parameter.expand(variables['cloud'])
cloud = clouds[0]
self.p = Provider(name=cloud)
self.secgroupname = "CM4TestSecGroup"
self.secgrouprule = {
"ip_protocol": "tcp",
"from_port": 8080,
"to_port": 8088,
"ip_range": "129.79.0.0/16"
}
self.testnode = None
print("\n")
def run(self,
script=None,
hosts=None,
username=None,
processors=4,
verbose=False):
results = []
if type(hosts) != list:
hosts = Parameter.expand(hosts)
for command in script.splitlines():
print(hosts, "->", command)
if command.startswith("#") or command.strip() == "":
pass
# print (command)
elif len(hosts) == 1 and hosts[0] == self.hostname:
os.system(command)
elif len(hosts) == 1 and hosts[0] != self.hostname:
host = hosts[0]
os.system(f"ssh {host} {command}")
else:
result = Host.ssh(hosts=hosts,
command=command,
username=username,
key="~/.ssh/id_rsa.pub",
processors=processors,
executor=os.system)
results.append(result)
if verbose:
pprint(results)
for result in results:
print(Printer.write(result, order=['host', 'stdout']))
return results
def sequence_remote(
led=None,
hosts=None,
username=None,
rate=None,
processors=3):
if led not in [1, 0]:
raise ValueError("Led number is wrong")
rate = float(rate or 0.5)
hosts = Parameter.expand(hosts)
for host in hosts:
LED.set_remote(
led=led,
value="0",
hosts=host,
username=username,
processors=processors)
time.sleep(rate)
LED.set_remote(
led=led,
value="1",
hosts=host,
username=username,
processors=processors)
time.sleep(rate)
return None
def main():
arguments = docopt(__doc__)
tags = Parameter.expand(arguments["VERSIONS"])
found = Shell.run("git tag").strip().splitlines()
# print (found)
for tag in tags:
if tag in found:
print(f"Removing tag {tag}")
script = [
f"git tag -d {tag}",
f"git push origin :refs/tags/{tag}"
]
if arguments["--dryrun"]:
print(" " + '\n '.join(script))
else:
try:
for line in script:
os.system(line)
Console.ok(f"{tag} deleted")
except:
Console.error ("Deletion failed")
else:
Console.error(f"{tag} does not exist")
def add(self, name=None):
"""
adds a key to a given group. If the group does not exist, it will be
created.
:param name:
:param keys:
:return:
"""
new_key = name
if type(name) == str:
new_key = Parameter.expand(name)
elif type(name) == list:
pass
else:
raise ValueError("key have wrong type")
# noinspection PyUnusedLocal
try:
entry = self.find(name=name)[0]
except Exception as e:
entry = {
'name': name
}
if name is not None:
old = list(entry['name'])
entry['name'] = list(set(new_key + old))
return self.update_dict_list([entry])
def get_filename(filename, hosts):
if filename is not None:
return filename
if type(hosts) == str:
hosts = Parameter.expand(hosts)
label = hosts[0]
return path_expand(f"~/.ssh/cluster_keys_{label}")
def setup(self):
variables = Variables()
print(variables['storage'])
self.service = Parameter.expand(variables['storage'])[0]
self.p = Provider(service=self.service)
self.sourcedir = path_expand("~/.cloudmesh/storage/test")
print()
def get_names(arguments, variables):
names = arguments["NAME"] or arguments["NAMES"] or arguments[
"--name"] or variables["vm"]
if names is None:
return None
else:
return Parameter.expand(names)
def setup(self):
variables = Variables()
self.service = Parameter.expand(variables['storage'])[0]
self.p = Provider(service=self.service)
self.sourcedir = os.path.expanduser(
"~/Documents/cloudmesh/storage/test")
print()
def delete(self, name=None):
"""
deletes the groups
:param name:
:param rules:
:return:
"""
delete_key = name
if type(name) == str:
delete_key = Parameter.expand(name)
elif type(name) == list:
pass
else:
raise ValueError("key have wrong type")
delete_key = set(delete_key)
entry = self.find(name=name)[0]
if name is not None:
old = set(entry['key'])
old -= delete_key
entry['key'] = list(old)
return entry
def add(self,
name=None,
services=None,
category=None):
# check if non and raise error
if type(services) == str:
services = Parameter.expand(services)
# cm = CmDatabase()
entry = {
'cm': {
"name": name,
"cloud": self.cloud,
"kind": self.kind
}
}
entry['members'] = [] # find in db
old = DictList(entry['members'])
entries = [{'name': service, 'kind': category} for
service in services]
for entry in old:
if entry not in entries:
entries.append(old[entry])
entry['members'] = entries
return [entry]
def delete(self, name=None, rules=None):
"""
deletes the groups
:param name:
:param rules:
:return:
"""
delete_rules = rules
if type(rules) == str:
delete_rules = Parameter.expand(rules)
elif type(rules) == list:
pass
else:
raise ValueError("rules have wrong type")
delete_rules = set(delete_rules)
entry = self.find(name=name)[0]
if rules is not None:
old = set(entry['rules'])
old -= delete_rules
entry['rules'] = list(old)
return entry
def collections(self, name=None, regex=None):
"""
the names of all collections
:param name: if set, only look at these collections instead of all
collections
:param regex: a regular expression on the names of the collections
:return: list of names of all collections
Example:
collections = cm.collections(regex=".*-vm")
"""
MongoDBController().start_if_not_running()
names = None
if name:
if type(name) == list:
names = name
else:
names = Parameter.expand(name)
else:
names = self.db.collection_names()
if regex:
r = re.compile(regex)
_names = list(filter(r.match, names))
names = _names
return names
def suspend(self, names=None):
"""
NOT YET IMPLEMENTED.
suspends the node with the given name.
:param name: the name of the node
:return: The dict representing the node
"""
HEADING(c=".")
names = Parameter.expand(names)
nodes = self.list(raw=True)
for node in nodes:
if node.name in names:
r = self.cloudman.ex_stop_node(self._get_node(node.name),
deallocate=False)
print(r)
self.cloudman.destroy_node(node)
raise NotImplementedError
#
# should return the updated names dict, e.g. status and so on
#
return None
def __init__(self, hostname=None, ip=None, dryrun=False, no_diagram=False):
self.dryrun = dryrun or False
self.hostnames_str = hostname
self.ips_str = ip
self.hostnames = hostnames = hostname or "red,red[01-02]"
self.ips = ips = ip or "10.0.0.[1-3]"
self.ssid = get_ssid()
self.imaged = ""
self.wifipassword = ""
self.no_diagram = no_diagram
hostnames = Parameter.expand(hostnames)
manager, workers = Host.get_hostnames(hostnames)
if workers is None:
n = 1
else:
n = len(workers) + 1
if ip is None:
ips = Parameter.expand(f"10.1.1.[1-{n}]")
else:
ips = Parameter.expand(ips)
# cluster_hosts = tuple(zip(ips, hostnames))
self.key = path_expand("~/.ssh/id_rsa.pub")
banner("Parameters", figlet=True)
print("Manager: ", manager)
print("Workers: ", workers)
print("IPS: ", ips)
print("Key: ", self.key)
print("Dryrun: ", self.dryrun)
self.manager = manager
self.workers = workers
self.ips = ips
self.load_data()
if not self.no_diagram:
self.create_diag(self.manager)
self.create_layout()
# sg.change_look_and_feel('SystemDefault')
self.window = sg.Window('Cloudmesh Pi Burn', self.layout, resizable=True, size=window_size)
def get_names(arguments, variables):
names = arguments["NAME"] or arguments["NAMES"] or arguments[
"--name"] or variables["vm"]
if names is None:
Console.error("you need to specify a vm")
return None
else:
return Parameter.expand(names)
