def test_cms_info_cloud(self):
HEADING()
cmd = "cms info"
result = Shell.run(cmd)
print(result)
assert "cloudmesh.cloud" in result
def run(label, command):
result = Shell.run_timed(label, command, service=cloud)
print(result)
return result
def az(self, command):
print(command)
r = Shell.execute(command, shell=True)
data = json.loads(r)
return data
from cloudmesh.common.Shell import Shell
result = Shell.execute('pwd') # Printing working directory
print("result: ", result)
result1 = Shell.execute('ls', ["-l"])
print("result1: ", result1)
result2 = Shell.ls("-a", "-u", "-x") # build in functions of Shell
print("result2: ", result2)
def run(command):
parameter = command.split(" ")
shell_command = parameter[0]
args = parameter[1:]
result = Shell.execute(shell_command, args)
return str(result)
def info(self, name=None):
"""
gets the information of a node with a given name
:param name:
:return: The dict representing the node including updated status
"""
arg = dotdict()
arg.name = name
config = Config()
cloud = "vagrant" # TODO: must come through parameter or set cloud
arg.path = config.data["cloudmesh"]["cloud"][cloud]["default"]["path"]
arg.directory = os.path.expanduser("{path}/{name}".format(**arg))
data = {
"cm": {
"name": name,
"directory": arg.directory,
"path": arg.path,
"cloud": cloud,
"status": "unkown"
}
}
result = None
result = Shell.execute("vagrant", ["ssh-config"],
cwd=arg.directory,
traceflag=False,
witherror=False)
if result is None:
data_vagrant = None
data["cm"]["status"] = "poweroff"
else:
print(result)
lines = result.split("\n")
data_vagrant = {}
for line in lines:
attribute, value = line.strip().split(" ", 1)
if attribute == "IdentityFile":
value = value.replace('"', '')
data_vagrant[attribute] = value
vms = Shell.execute('VBoxManage', ["list", "vms"]).split("\n")
#
# find vm
#
vbox_name_prefix = "{name}_{name}_".format(**arg)
# print (vbox_name_prefix)
details = None
for vm in vms:
vm = vm.replace("\"", "")
vname = vm.split(" {")[0]
if vname.startswith(vbox_name_prefix):
details = Shell.execute(
"VBoxManage", ["showvminfo", "--machinereadable", vname])
# print (details)
break
vbox_dict = self._convert_assignment_to_dict(details)
# combined = {**data, **details}
# data = combined
if data_vagrant is not None:
data["vagrant"] = data_vagrant
data["vbox"] = vbox_dict
if "VMState" in vbox_dict:
data["cm"]["status"] = vbox_dict["VMState"]
return data
def do_vbox(self, args, arguments):
"""
::
Usage:
vbox version [--format=FORMAT]
vbox image list [--format=FORMAT]
vbox image find KEYWORDS...
vbox image add NAME
vbox image delete NAME
vbox vm info NAME
vbox vm list [--format=FORMAT] [-v]
vbox vm delete NAME
vbox vm ip [NAME] [--all]
vbox vm create [NAME] ([--memory=MEMORY] [--image=IMAGE] [--port=PORT] [--script=SCRIPT] | list)
vbox vm boot [NAME] ([--memory=MEMORY] [--image=IMAGE] [--port=PORT] [--script=SCRIPT] | list)
vbox vm ssh [NAME] [-e COMMAND]
"""
arguments.format = arguments["--format"] or "table"
arguments.verbose = arguments["-v"]
arguments.all = arguments["--all"]
#
# ok
#
def list_images():
images = VboxProvider().list_images()
_LIST_PRINT(images,
arguments.format,
order=["name", "provider", "date"])
#
# ok
#
# noinspection PyShadowingNames
def image_command(func):
try:
images = func(arguments.NAME)
print(images)
list_images()
except Exception as e:
print(e)
return ""
#
# ok
#
if arguments.version:
versions = {
"cm": {
"attribute": "cm",
"description": "Cloudmesh vbox Version",
"version": __version__
},
"vbox": {
"attribute": "vbox",
"description": "Vagrant Version",
"version": "TBD" # cloudmesh.vbox.api.version(),
}
}
result = Printer.write(
versions,
order=["attribute", "version", "description"],
output=arguments.format)
print(result)
#
# ok
#
elif arguments.image and arguments.list:
list_images()
#
# ok
#
elif arguments.image and arguments.delete:
image_command(VboxProvider().delete_image)
#
# ok
#
elif arguments.image and arguments.put:
image_command(VboxProvider().add_image)
#
# ok
#
elif arguments.image and arguments.find:
VboxProvider().find_image(arguments.KEYWORDS)
return ""
#
# ok, but only vagrant details
#
elif arguments.vm and arguments.list:
provider = VboxProvider().nodes()
_LIST_PRINT(provider,
arguments.format,
order=["name", "state", "id", "provider", "directory"])
return ""
#
# unclear: this function is unclear
#
elif arguments.create and arguments.list:
result = Shell.cat("{NAME}/Vagrantfile".format(**arguments))
if result is not None:
print(result)
return ""
elif arguments.create:
d = defaults()
print("LLLL", d)
arguments.memory = arguments["--memory"] or d.memory
arguments.image = arguments["--image"] or d.image
arguments.script = arguments["--script"] or d.script
arguments.port = arguments["--port"] or d.port
server = VboxProvider()
server.create(**arguments)
elif arguments.info:
# arguments.NAME
d = VboxProvider().info(name=arguments.NAME)
result = Printer.write(d, output=arguments.format)
print(result)
elif arguments.ip:
data = []
result = VboxProvider().execute(arguments.NAME, "ifconfig")
if result is not None:
lines = result.splitlines()[:-1]
for line in lines:
if "inet addr" in line:
line = line.replace("inet addr", "ip")
line = ' '.join(line.split())
_adresses = line.split(" ")
address = {}
for element in _adresses:
attribute, value = element.split(":")
address[attribute] = value
data.append(address)
if arguments.all:
d = {}
i = 0
for e in data:
d[str(i)] = e
i = i + 1
result = Printer.attribute(d, output=arguments.format)
print(result)
else:
for element in data:
ip = element['ip']
if ip == "127.0.0.1" or ip.startswith("10."):
pass
else:
print(element['ip'])
elif arguments.boot:
d = defaults()
arguments.memory = arguments["--memory"] or d.memory
arguments.image = arguments["--image"] or d.image
arguments.script = arguments["--script"] or d.script
arguments.port = arguments["--port"] or d.port
node = VboxProvider().boot(name=arguments.NAME,
memory=arguments.memory,
image=arguments.image,
script=arguments.script,
port=arguments.port)
elif arguments.delete:
result = VboxProvider().delete(name=arguments.NAME)
print(result)
elif arguments.ssh:
if arguments.COMMAND is None:
os.system("cd {NAME}; vbox ssh {NAME}".format(**arguments))
else:
result = VboxProvider().execute(arguments.NAME,
arguments.COMMAND)
if result is not None:
lines = result.splitlines()[:-1]
for line in lines:
print(line)
else:
print("use help")
result = ""
return result
def do_init(self, args, arguments):
"""
::
Usage:
init [CLOUD] [--debug]
init yaml
Description:
Initializes cloudmesh while using data from
~/.cloudmesh/cloudmesh.yaml.
If no cloud is specified a number of local collections are
created. If a cloud is specified it also uploads the
information about images, flavors, vms. It also uploads the
security groups defined by default to the cloud.
Bug:
cms init
cms init
On Windows you have to run the cms init command twice upon
first installation
"""
if arguments.CLOUD == "yaml":
config = Config()
location = path_expand("~/.cloudmesh/cloudmesh.yaml")
path = Path(location)
if path.is_file():
print()
if yn_choice(
"The file ~/.cloudmesh/cloudmesh.yaml exists, do you wnat to overwrite it",
default='n'):
config.fetch()
print()
Console.ok("File cloudmesh.yaml downloaded from Github")
else:
print()
Console.warning("Download canceled")
print()
else:
variables = Variables()
config = Config()
try:
print("MongoDB stop")
MongoDBController().stop()
except:
Console.ok("MongoDB is not running. ok")
machine = platform.lower()
location = path_expand(config[
f'cloudmesh.data.mongo.MONGO_DOWNLOAD.{machine}.MONGO_PATH'])
try:
shutil.rmtree(location)
print("MongoDB folder deleted")
except:
Console.error(f"Could not delete {location}")
if platform == 'win32':
Console.error(f"Please try to run cms init again ... ")
exit(1)
config = Config()
user = config["cloudmesh.profile.user"]
secgroup = "flask"
print("Set key")
if user == "TBD":
Console.error(
"the user is not set in the yaml file for cloudmesh.profile.user")
sys.exit()
variables["key"] = user
Console.ok("Config Security Initialization")
Shell.execute("cms", ["config", "secinit"])
print("MongoDB create")
os.system("cms admin mongo create")
os.system("cms admin mongo start")
os.system("cms sec load")
if arguments.CLOUD is not None:
cloud = arguments.CLOUD
variables['cloud'] = cloud
os.system(f"cms key upload {user} --cloud={cloud}")
os.system(f"cms flavor list --refresh")
os.system(f"cms image list --refresh")
os.system(f"cms vm list --refresh")
os.system(f"cms sec group load {secgroup} --cloud={cloud}")
os.system(f"cms set secgroup={secgroup}")
if arguments.debug:
variables['debug'] = True
variables['timer'] = 'on'
variables['trace'] = True
variables['verbose'] = '10'
print()
print("Variables")
print()
for name in variables:
value = variables[name]
print(f" {name}={value}")
def _execute_src_dest(self, cmd, src, dest=None):
if dest is None:
return Shell.execute('ampy', ['--port', self.port, cmd, src])
else:
return Shell.execute('ampy', ['--port', self.port, cmd, src, dest])
# fa19-516-163 E.Cloudmesh.Common.3
# Cloudmesh imports
from cloudmesh.common.Shell import Shell
from cloudmesh.common.util import banner
# Ping a host
banner("Ping API: itpeople-api.apps.iu.edu")
ping = Shell.ping("itpeople-api.apps.iu.edu")
# Print the result
banner("Ping result")
print(ping)
from cloudmesh.common.Shell import Shell
from pprint import pprint
print("Starting debug")
result = Shell.execute("cms", ["storage", "--storage=local", "list", "a"])
pprint(result)
def check(self, path=None):
# bug: path not needed
error = False
# path = path_expand(path or self.location.config())
path = path_expand(path or self.location.config())
#
# bug path not passed along ;-) we can just remove it
#
config = Config()
banner("Check for CLOUDMESH_CONFIG_DIR")
if os.path.isfile(path):
print("Config found in:", path)
if "CLOUDMESH_CONFIG_DIR" in os.environ:
directory = os.environ("CLOUDMESH_CONFIG_DIR")
print("CLOUDMESH_CONFIG_DIR={directory}")
config_path = str(Path(directory) / "cloudmesh.yaml")
if os.path.isfile(path):
print("Config found in:", path)
else:
Console.error(f"File {config_path} not found.")
if path != config_path:
Console.warning("You may have two cloudmesh.yaml file.")
Console.warning("We use: {config_path is use}")
banner("Check Version")
dist_version = config.version()
yaml_version = config["cloudmesh.version"]
if dist_version == yaml_version:
Console.ok(f"The version is {dist_version}")
else:
Console.error("Your version do not match")
print()
print("Found ~/.cloudmesh/cloudmesh.yaml:", yaml_version)
print("Please update to version :", dist_version)
print("")
print("See also: ")
print()
print(
" https://github.com/cloudmesh/cloudmesh-configuration/blob/master/cloudmesh/configuration/etc/cloudmesh.yaml"
)
banner("Check for TAB Characters")
error = Config.check_for_tabs(path)
if not error:
Console.ok("OK. No TABs found")
banner("yamllint")
try:
import yamllint
options = \
'-f colored ' \
'-d "{extends: relaxed, ""rules: {line-length: {max: 256}}}"'
r = Shell.live('yamllint {options} {path}'.format(**locals()))
if 'error' in r or 'warning' in r:
print(70 * '-')
print(" line:column description")
print()
else:
Console.ok("OK. No issues found")
print()
except:
Console.error("Could not execute yamllint. Please add with")
Console.error("pip install yamllint")
def do_host(self, args, arguments):
"""
::
Usage:
host scp NAMES SOURCE DESTINATION [--dryrun]
host ssh NAMES COMMAND [--dryrun] [--output=FORMAT]
host config NAMES --ips=IPS [--user=USER] [--key=PUBLIC]
host config --proxy=PROXY NAMES [--user=USER] [--append] [--local=no] [--StrictHostKeyChecking=no] [--cluster=name]
host config NAMES [--user=USER] [--append] [--local=no] [--StrictHostKeyChecking=no] [--cluster=name]
host find [NAMES] [--user=USER] [--table|--json] [--verbose]
host check NAMES [--user=USER] [--key=PUBLIC]
host key create NAMES [--user=USER] [--dryrun] [--output=FORMAT]
host key list NAMES [--output=FORMAT]
host key setup NAMES
host key gather NAMES [--authorized_keys] [FILE]
host key scatter NAMES [FILE] [--user=USER]
host key add NAMES [FILE]
host key delete NAMES [FILE]
host key access NAMES [FILE] [--user=USER]
host tunnel create NAMES [--port=PORT]
host mac NAMES [--eth] [--wlan] [--output=FORMAT]
host setup WORKERS [LAPTOP]
host shutdown NAMES
host reboot NAMES
host adduser NAMES USER
host passwd NAMES USER
host addsudo NAMES USER
host deluser NAMES USER
host ping NAMES
host info NAMES
This command does some useful things.
Arguments:
FILE a file name
Options:
--dryrun shows what would be done but does not execute
--output=FORMAT the format of the output
--port=PORT starting local port for tunnel assignment
--local=no do not append .local to manager hostname [default: yes]
--user=USER username for manager and workers [default: pi]
--ips=IPS ip addresses of the manager and workers
--StrictHostKeyChecking=no if set to yes, strict host checking is enforced [default: no]
--ProxyJump=no if set to yes, a proxyjump is performed for each worker through the manager [default: yes]
Description:
host scp NAMES SOURCE DESTINATION
Uses scp to transfer Source to NAMES:DESTINATION.
host ssh NAMES COMMAND
runs the command on all specified hosts
Example:
ssh red[01-10] \"uname -a\"
host key create NAMES
create a ~/.ssh/id_rsa and id_rsa.pub on all hosts specified
Example:
ssh key create "red[01-10]"
host key list NAMES
list all id_rsa.pub keys from all hosts specifed
Example:
ssh key list red[01-10]
host key gather HOSTS FILE
gathers all keys from file FILE including the one from localhost.
ssh key gather "red[01-10]" keys.txt
host key scatter HOSTS FILE [--user=USER]
copies all keys from file FILE to authorized_keys on all hosts,
but also makes sure that the users ~/.ssh/id_rsa.pub key is in
the file. If provided the optional user, it will add the keys to
that user's .ssh directory. This is often required when
adding a new user in which case HOSTS should still a sudo
user with ssh currently enabled.
1) adds ~/.id_rsa.pub to the FILE only if its not already in it
2) removes all duplicated keys
Example:
ssh key scatter "red[01-10]"
ssh key scatter pi@red[01-10] keys.txt --user=alice
host key add NAMES FILE
Adds all keys in FILE into the authorized_keys of NAMES.
Example:
cms host key add worker001 ~/.ssh/id_rsa.pub
host key delete NAMES FILE
Deletes all keys in fILE from authorized_keys of NAMES if they exist.
Example
cms host key delete worker001 ~/.ssh/id_rsa.pub
host key scp NAMES FILE
copies all keys from file FILE to authorized_keys on all hosts
but also makes sure that the users ~/.ssh/id_rsa.pub key is in
the file and removes duplicates, e.g. it calls fix before upload
Example:
ssh key list red[01-10] > pubkeys.txt
ssh key scp red[01-10] pubkeys.txt
host config NAMES IPS [--user=USER] [--key=PUBLIC]
generates an ssh config file tempalte that can be added to your
.ssh/config file
Example:
cms host config "red,red[01-03]" "198.168.1.[1-4]" --user=pi
host check NAMES [--user=USER] [--key=PUBLIC]
This command is used to test if you can login to the specified
hosts. It executes the hostname command and compares it.
It provides a table with a sucess column
cms host check "red,red[01-03]"
+-------+---------+--------+
| host | success | stdout |
+-------+---------+--------+
| red | True | red |
| red01 | True | red01 |
| red02 | True | red02 |
| red03 | True | red03 |
+-------+---------+--------+
host tunnel create NAMES [--port=PORT]
This command is used to create a persistent local port
forward on the host to permit ssh tunnelling from the wlan to
the physical network (eth). This registers an autossh service in
systemd with the defualt port starting at 8001.
Example:
cms host tunnel create red00[1-3]
host mac NAMES
returns the list of mac addresses of the named pis.
host setup WORKERS [LAPTOP]
Executes the following steps
cms bridge create --interface='wlan0'
cms host key create red00[1-3]
cms host key gather red00[1-3],[email protected] keys.txt
cms host key scatter red00[1-3],localhost keys.txt
rm keys.txt
cms host tunnel create red00[1-3]
host shutdown NAMES
Shutsdown NAMES with `sudo shutdown -h now`. If localhost in
names, it is shutdown last.
host reboot NAMES
Reboots NAMES with `sudo reboot`. If localhost in names,
it is rebooted last.
host adduser NAMES USER
Adds a user with user name USER to the hosts identified by
NAMES. Password is disabled, see host passwd to enable.
host addsudo NAMES USER
Adds sudo rights to USER at NAMES
host passwd NAMES USER
Changes the password for USER at NAMES
host deluser NAMES USER
Deleted USER from NAMES. Home directory will be removed.
host config proxy PROXY NAMES
This adds to your ~/.ssh/config file a ProxyJump
configuration to reach NAMES via PROXY. This is useful when
the PROXY is acting as a network bridge for NAMES to your
current device.
Example:
cms host config proxy [email protected] red00[1-2]
"""
def _print(results):
arguments.output = arguments.output or 'table'
if arguments.output in ['table', 'yaml']:
print(
Printer.write(
results,
order=['host', 'success', 'stdout', 'stderr'],
output=arguments.output))
else:
pprint(results)
def _print_pis(results):
arguments.output = arguments.output or 'table'
if arguments.output in ['table', 'yaml']:
print(
Printer.write(results,
order=[
'name', 'ip', 'user', 'os', 'mac',
'model', 'memory', 'serial', ".local"
],
output=arguments.output))
# not printed "revision"
# not printed "hardware"
else:
pprint(results)
def get_filename(filename, hosts):
if filename is not None:
return filename
if type(hosts) == str:
hosts = Parameter.expand(hosts)
label = hosts[0]
return path_expand(f"~/.ssh/cluster_keys_{label}")
map_parameters(arguments, 'eth', 'wlan'
'dryrun', 'output', 'user', 'port', 'append',
'StrictHostKeyChecking', 'local', 'proxy', 'ips',
'cluster')
dryrun = arguments.dryrun
# VERBOSE(arguments)
if dryrun:
VERBOSE(arguments)
if arguments.info:
names = Parameter.expand(arguments.names)
# check if .local
# check if mesh network
# check if static network
# use arp - a di find hosts ips
# if linux
# dig +short -x 192.168.50.1
Console.error("Not yet Implemented")
elif arguments.find:
verbose = arguments["--verbose"]
names = Parameter.expand(arguments.NAMES)
# temporary so we can easy modify while not needing to update cloudmesh.common
from cloudmesh.host.network import PiNetwork
network = PiNetwork()
pis = network.find_pis(user=arguments.user, verbose=verbose)
if arguments["--json"]:
print(pis)
else:
_print_pis(pis)
elif arguments.mac:
names = Parameter.expand(arguments.NAMES)
if not arguments.eth and not arguments.wlan:
arguments.eth = True
arguments.wlan = True
eth = 'cat /sys/class/net/eth0/address'
wlan = 'cat /sys/class/net/wlan0/address'
if arguments.eth:
results = Host.ssh(hosts=names,
command=eth,
username=arguments.user)
print("eth0:")
_print(results)
if arguments.wlan:
results = Host.ssh(hosts=names,
command=wlan,
username=arguments.user)
print("wlan0:")
_print(results)
elif arguments.setup:
HostCreate.setup(workers=arguments.WORKERS,
laptop=arguments.LAPTOP)
elif arguments.scp and not arguments.key:
result = Host.put(hosts=arguments.NAMES,
source=arguments.SOURCE,
destination=arguments.DESTINATION)
_print(result)
elif arguments.ping:
names = Parameter.expand(arguments.NAMES)
# print (names)
results = Host.ping(hosts=names)
_print(results)
elif arguments.ssh:
names = Parameter.expand(arguments.NAMES)
# print (names)
results = Host.ssh(hosts=names, command=arguments.COMMAND)
_print(results)
elif arguments.key and arguments.create:
results = Host.ssh_keygen(hosts=arguments.NAMES,
username=arguments.user,
dryrun=dryrun)
_print(results)
elif arguments.key and arguments.list:
names = Parameter.expand(arguments.NAMES)
results = Host.ssh(hosts=names,
command='cat .ssh/id_rsa.pub',
username=arguments.user)
_print(results)
elif arguments.key and arguments.add:
filename = get_filename(arguments.NAMES)
if not os.path.isfile(filename):
Console.error(f"Cannot find file {filename}")
return
# Copy to temp location
Host.put(hosts=arguments.NAMES,
source=filename,
destination="~/.ssh/key.tmp")
# Execute append command and remove command
command = 'cat ~/.ssh/key.tmp >> ~/.ssh/authorized_keys && rm ~/.ssh/key.tmp'
Host.ssh(hosts=arguments.NAMES, command=command)
elif arguments.key and arguments.delete:
Console.ok("key delete")
filename = get_filename(arguments.FILE, arguments.NAMES)
if not os.path.isfile(filename):
Console.error(f"Cannot find file {filename}")
return
# Copy to temp location
remote_temp = "~/.ssh/key.tmp"
Host.put(hosts=arguments.NAMES,
source=filename,
destination=remote_temp)
# grep can read multiple patterns from a file, one per line. Combine with
# the options -v to output non-matching lines, and -F to match strings
# instead of regex and -x to require that the whole line matches.
command = f"""grep -Fvx -f {remote_temp} ~/.ssh/authorized_keys >remaining_keys && \
mv remaining_keys ~/.ssh/authorized_keys && \
rm {remote_temp}"""
Host.ssh(hosts=arguments.NAMES, command=command)
Console.ok(f"Delete keys from {filename} on {arguments.NAMES}")
elif arguments.key and arguments.setup:
label = Parameter.expand(arguments.NAMES)[0]
filename = get_filename(arguments.FILE, arguments.NAMES)
directory = os.path.dirname(filename)
if directory:
Shell.mkdir(directory)
output = Host.gather_keys(username=arguments.user,
hosts=arguments.NAMES,
filename="~/.ssh/id_rsa.pub",
key="~/.ssh/id_rsa",
processors=3,
dryrun=False)
with open(filename, "w") as f:
f.write(output)
# scatter
# place .ssh/config a trict host check to no
elif arguments.key and arguments.gather:
output = Host.gather_keys(username=arguments.user,
hosts=arguments.NAMES,
filename="~/.ssh/id_rsa.pub",
key="~/.ssh/id_rsa",
processors=3,
dryrun=False)
VERBOSE(arguments)
filename = get_filename(arguments.FILE, arguments.NAMES)
print(output)
banner(f"Writing Keys to file {filename}")
directory = os.path.dirname(filename)
print('command directory', directory)
if directory:
Shell.mkdir(directory)
if os.path.isfile(filename) and yn_choice(
f'{filename} is not empty. Do you wish to overwrite it? (If no you will append).'
):
with open(filename, "w") as f:
f.write(output)
else:
with open(filename, "a") as f:
f.write(output)
elif arguments.key and arguments.scatter:
#
# this should be a function in Host
#
filename = get_filename(arguments.FILE, arguments.NAMES)
names = arguments.NAMES
user = arguments.user
if not os.path.isfile(filename):
Console.error("The file does not exist")
return ""
if not user:
result = Host.put(hosts=names,
source=filename,
destination=".ssh/authorized_keys")
_print(result)
else:
Console.info('SCP to ./temp_authorzied_keys_temp')
result = Host.put(hosts=names,
source=filename,
destination="temp_authorized_keys_temp")
_print(result)
Console.info(f'Mkdir /home/{user}/.ssh if not exist')
command = f'sudo mkdir -p /home/' \
f'{user}/.ssh/'
result = Host.ssh(hosts=names, command=command)
_print(result)
Console.info(f'Chown /home/{user}/.ssh to {user}')
command = f'sudo chown {user}:{user} /home/' \
f'{user}/.ssh/'
result = Host.ssh(hosts=names, command=command)
_print(result)
Console.info(f'Chmod /home/{user}/.ssh to 700')
command = f'sudo chmod 700 /home/' \
f'{user}/.ssh/'
result = Host.ssh(hosts=names, command=command)
_print(result)
Console.info(f'Mv temp_authorized_keys_temp to /home/'
f'{user}/.ssh/authorized_keys')
command = f'sudo mv temp_authorized_keys_temp /home/' \
f'{user}/.ssh/authorized_keys'
result = Host.ssh(hosts=names, command=command)
_print(result)
Console.info(f'Chown /home/{user}/.ssh/authorized_keys to '
f'{user}')
command = f'sudo chown {user}:{user} /home/' \
f'{user}/.ssh/authorized_keys'
result = Host.ssh(hosts=names, command=command)
_print(result)
elif arguments.key and arguments.access:
#
# this should be a function in Host
#
names = arguments.NAMES
user = arguments.user
filename = arguments.FILE
temp = path_expand("~/.cloudmesh/temp_config")
if filename:
config = readfile(filename)
else:
config = textwrap.dedent("""
Host *
StrictHostKeyChecking no
""").strip()
writefile(temp, config)
if not os.path.isfile(temp):
Console.error("The file does not exist")
return ""
if not user:
result = Host.put(hosts=names,
source=temp,
destination=".ssh/config")
_print(result)
else:
Console.info(f'Mkdir /home/{user}/.ssh if not exist')
command = f'sudo mkdir -p /home/' \
f'{user}/.ssh/'
result = Host.ssh(hosts=names, command=command)
_print(result)
Console.info('SCP to ./temp_config')
result = Host.put(hosts=names,
source=temp,
destination=".ssh/config")
_print(result)
Console.info(f'Chown /home/{user}/.ssh to {user}')
command = f'sudo chown {user}:{user} /home/' \
f'{user}/.ssh/'
result = Host.ssh(hosts=names, command=command)
_print(result)
Console.info(f'Chmod /home/{user}/.ssh to 700')
command = f'sudo chmod 700 /home/' \
f'{user}/.ssh/'
result = Host.ssh(hosts=names, command=command)
_print(result)
elif arguments.config and arguments.ips:
print("NNNNNNNN")
key = arguments.key or "~/.ssh/id_rsa.pub"
result = Host.config(hosts=arguments.NAMES,
ips=arguments.IPS,
username=arguments.user,
key=key)
print(result)
"""
host config NAMES --ips=IPS [--user=USER] [--key=PUBLIC]
host config --proxy=PROXY NAMES [--user=USER] [--append] [--local=no] [--StrictHostKeyChecking=no]
host config NAMES [--user=USER] [--append] [--local=no] [--StrictHostKeyChecking=no]
"""
elif arguments.config:
if str_bool(arguments.local):
local_str = ".local"
else:
local_str = ""
if str_bool(arguments.StrictHostKeyChecking):
strict_host_str = "yes"
else:
strict_host_str = "no"
names = Parameter.expand(arguments.NAMES)
user = arguments.user
if arguments.cluster:
cluster = arguments.cluster
else:
# take the first name and remove spaces
cluster = ''.join([i for i in names[0] if not i.isdigit()])
ssh_config_output = ""
ssh_config_output = f'\n##### CLOUDMESH PROXY CONFIG {cluster} #####\n\n'
if arguments.proxy:
proxy_host = arguments.proxy
proxy_jump = f' ProxyJump {proxy_host}\n'
ssh_config_output += \
f'Host {proxy_host}\n' \
f' HostName {proxy_host}{local_str}\n' \
f' User {user}\n' \
f' PreferredAuthentications publickey\n' + \
f' StrictHostKeyChecking {strict_host_str}\n'
ssh_config_output += '\n'
else:
proxy_jump = ""
"""
ssh_config_output = f'\n##### CLOUDMESH PROXY CONFIG {cluster} #####\n\n'\
f'Host {proxy_host}\n' \
f' HostName {proxy_host}{local_str}\n' \
f' User {user}\n' \
f' StrictHostKeyChecking {strict_host_str}\n\n'
"""
### the local_str in the hostname may be wrong since its not manager
for name in names:
ssh_config_template = f'Host {name}\n' \
f' HostName {name}{local_str}\n' \
f' User {user}\n' \
f' PreferredAuthentications publickey\n' + \
f' StrictHostKeyChecking {strict_host_str}\n' + \
proxy_jump
ssh_config_template += '\n'
ssh_config_output += ssh_config_template
ssh_config_output += f'##### CLOUDMESH PROXY CONFIG {cluster} #####\n'
print('Adding to ~/.ssh/config')
print(ssh_config_output)
if not os.path.exists(path_expand('~/.ssh/config')):
with open(path_expand('~/.ssh/config'), 'w+') as f:
f.write(ssh_config_output)
else:
f = open(path_expand('~/.ssh/config'), 'r')
lines = f.readlines()
f.close()
with open(path_expand('~/.ssh/config'), 'w+') as f:
if f'##### CLOUDMESH PROXY CONFIG {cluster} #####\n' in lines:
start = lines.index(
f'##### CLOUDMESH PROXY CONFIG {cluster} #####\n')
lines.reverse()
end = lines.index(
f'##### CLOUDMESH PROXY CONFIG {cluster} #####\n')
end = len(lines) - end - 1
lines.reverse()
original_config = lines[start:end + 1]
del lines[start:end + 1]
f.writelines(lines)
if arguments.append:
f.writelines(original_config)
f.write(ssh_config_output)
else:
f.write(ssh_config_output)
else:
f.writelines(lines)
f.write(ssh_config_output)
f.close()
elif arguments.check:
key = arguments.key or "~/.ssh/id_rsa.pub"
result = Host.check(hosts=arguments.NAMES,
username=arguments.user,
key=key)
for entry in result:
entry['success'] = entry['stdout'] == entry['host']
_print(result)
elif arguments.tunnel and arguments.create:
wlan_ip = Shell.run("hostname -I | awk '{print $2}'").strip()
print(f'\nUsing wlan0 IP = {wlan_ip}')
hostname = Shell.run("hostname").strip()
print(f'Using cluster hostname = {hostname}')
names = Parameter.expand(arguments.NAMES)
port = arguments.port or "8001"
ssh_config_output = f'Host {hostname}\n' \
f' HostName {hostname}.local\n' \
f' User pi\n\n'
for name in names:
service_name = f"autossh-{name}.service"
service_template = "[Unit]\n" \
f"Description=AutoSSH tunnel service to {name} on local port " \
f"{port}\n" \
"After=multi-user.target\n\n" \
"[Service]\n" \
"User=pi\n" \
"Group=pi\n" \
'Environment="AUTOSSH_GATETIME=0"\n' \
'ExecStart=/usr/bin/autossh -M 0 -o "ServerAliveInterval 30" ' \
'-o "ServerAliveCountMax 3" -i ' \
'/home/pi/.ssh/id_rsa -NL ' \
f'{wlan_ip}:{port}:localhost:22 p' \
f'i@{name}\n\n' \
"[Install]\n" \
"WantedBy=multi-user.target"
ssh_config_template = f'Host {name}\n' \
f' HostName {hostname}.local\n' \
f' User pi\n' \
f' Port {port}\n\n'
ssh_config_output += ssh_config_template
Sudo.writefile(f'/etc/systemd/system/{service_name}',
service_template)
port = str(int(port) + 1)
os.system('sudo systemctl daemon-reload')
for name in names:
servicename = f"autossh-{name}.service"
os.system(f'sudo systemctl start {servicename}')
os.system(f'sudo systemctl enable {servicename}')
print('\nTunnels created.\n\nPlease place the following in your '
'remote machine\'s (i.e. laptop) ~/.ssh/config file to '
'alias simple ssh access (i.e. ssh red001).')
banner('copy to ~/.ssh/config on remote host (i.e laptop)')
print(ssh_config_output)
elif arguments.shutdown or arguments.reboot:
if arguments.shutdown:
command = 'sudo shutdown -h now'
elif arguments.reboot:
command = 'sudo reboot'
names = Parameter.expand(arguments.NAMES)
hostname = Shell.run("hostname").strip()
localhost = None
if "localhost" in names:
names.remove("localhost")
localhost = True
if hostname in names:
names.remove(hostname)
localhost = True
manager, workers = Host.get_hostnames(names)
if workers:
Console.info(f'Executing `{command}` for {workers}')
Host.ssh(hosts=workers, command=command)
if manager:
Console.info(f'Executing `{command}` for {manager}')
Host.ssh(hosts=manager, command=command)
#_print(results)
# results can be misleading becuase there is a race between the
# shutdown and the error code being returned from the ssh processes.
if localhost:
os.system(command)
elif arguments.adduser:
names = Parameter.expand(arguments.NAMES)
user = arguments.USER
localhost = None
if 'localhost' in names:
localhost = 'localhost'
elif platform.node() in names:
localhost = platform.node()
if localhost in names:
print('\nAdding user to localhost')
result = Shell.run(f'sudo adduser {user} '
f'--disabled-password '
f'--gecos "" ')
print(result)
names.remove(localhost)
if len(names) > 0:
command = f"sudo adduser {user} --disabled-password --gecos ',' "
results = Host.ssh(hosts=names, command=command)
_print(results)
elif arguments.passwd:
names = Parameter.expand(arguments.NAMES)
user = arguments.USER
localhost = None
if 'localhost' in names:
localhost = 'localhost'
elif platform.node() in names:
localhost = platform.node()
if localhost in names:
print("\nSetting password on localhost, please provide user "
"password")
result = os.system(f'sudo passwd {user}')
print(result)
names.remove(localhost)
if len(names) > 0:
print("\nSetting password on remote hosts, please enter user "
"password\n")
password = getpass("Please enter the user password:"******"{password}\n{password}" | sudo passwd {user}'
results = Host.ssh(hosts=names, command=command)
_print(results)
elif arguments.addsudo:
names = Parameter.expand(arguments.NAMES)
user = arguments.USER
localhost = None
if 'localhost' in names:
localhost = 'localhost'
elif platform.node() in names:
localhost = platform.node()
if localhost in names:
print('\nAdding user to sudo group on localhost')
result = Shell.run(f'sudo adduser {user} sudo')
print(result)
names.remove(localhost)
if len(names) > 0:
command = f'sudo adduser {user} sudo'
results = Host.ssh(hosts=names, command=command)
_print(results)
elif arguments.deluser:
names = Parameter.expand(arguments.NAMES)
user = arguments.USER
if 'localhost' in names:
localhost = 'localhost'
elif platform.node() in names:
localhost = platform.node()
if localhost in names:
print('\nDeleting user on localhost')
result = Shell.run(f'sudo userdel -r {user}')
print(result)
names.remove(localhost)
if len(names) > 0:
command = f'sudo userdel -r {user}'
results = Host.ssh(hosts=names, command=command)
_print(results)
return ""
def installed():
r = Shell.which("rpi-imager")
return r is not None
#!/usr/bin/env python3
# Josh Goodman sp20-516-220 E.Cloudmesh.Common.4
from cloudmesh.common.Shell import Shell
if __name__ == "__main__":
lines = Shell.grep('Josh', __file__)
print(f"The following lines match 'Josh' in {__file__}:\n\n{lines}")
def _execute(self, cmd, src):
return Shell.execute('ampy', ['--port', self.port, cmd, src])
def aws_delete():
print("setting cloud to aws")
Shell.execute("cms set cloud=aws", shell=True)
print("deleting austin-vm-aws_a1...")
Shell.execute("cms vm delete austin-vm-aws_a1", shell=True)
def do_cluster(self, args, arguments):
"""
::
Usage:
cluster test
cluster build --id=ID LABEL
cluster create --cloud=CLOUD --n=N LABEL
cluster add --id="[ID]" --all LABEL
cluster remove --id="[ID]" --all LABEL
cluster terminate --all LABEL
cluster info [--verbose=V] [LABEL]
This command allows you to create and interact with an available cluster of machines.
Arguments:
ID An existing machine ID to be reserved for the cluster.
LABEL The label for the cluster.
CLOUD Cloud platform to initialize VMs.
N Number of instances to request.
V Verbosity level.
Options:
--id Specify string containing list of IDs, comma delimited format "id1,id2,...,idx".
--cloud Specify cloud platform {AWS, Azure, Openstack}.
--n Specify number of VMs to initialize.
--all OPTIONAL. Overrides --id, will pass all machines as an argument.
--verbose OPTIONAL. Provides verbosity level for info.
Description:
cluster build --id=ID LABEL
Groups together existing machines and reserves them for cluster use. Pass a comma-delimited list of machine ID's as a string.
Pass --all to associate all available machines to cluster.
cluster create --cloud=CLOUD --n=N LABEL
Automatically requests VMs from the cloud service requested.
cluster add --id="[ID]" --all LABEL
Adds given machine IDs to cluster. Pass --all to associate all available machines to cluster.
cluster remove --id="[ID]" LABEL
Removes given machine IDs from cluster. Pass --all to disassociate all machines from cluster.
cluster terminate --all LABEL
Terminates all instances associated with the cluster, wipes cluster data. If --all is passed, terminates all running clusters.
cluster info --all --verbose=v [LABEL]
Retrieves cluster data and machine data associatred with cluster. Verbosity level 1 provides high-level cluster information
and list of machines associated. Verbosity level 2 provides cluster information, machine information and status. Verbosity
level 3 provides all available information.
"""
map_parameters(arguments, 'id', 'label', 'cloud', 'n', 'v', 'all',
'verbose')
# inv = Inventory()
# inv.read()
config = Config()
user = config["cloudmesh.profile.user"]
s = Shell()
cmdb = CmDatabase()
if arguments.test:
cmdb = CmDatabase()
virtual_clusters = cmdb.collection("cluster-virtual")
print(*[index for index in virtual_clusters.list_indexes()])
if arguments.build:
ids, label = arguments.id, arguments.LABEL
# Builds and stores a cluster connected to existing machine ids
machines = ids.split(",")
cluster_machines = []
for i, machine in enumerate(machines):
cluster_machines.append({
f"{machine}_{i}": {
"type": "cloud",
"cloud": None,
"status": "available",
"deployment": None
}
})
print(f"Adding the following machines to cluster-cloud {label}: ")
VERBOSE(cluster_machines)
collection = cmdb.collection("cluster-cloud")
collection.insert_one({label: cluster_machines})
# # TODO Revise to update to correct mongo create/update
# cmdb.UPDATE(clusters)
if arguments.create:
n, label, cloud = arguments.n, arguments.label, arguments.cloud
ids = [f"label_{i}" for i in range(n)].join(",")
starting = [
s.run(f"cms vm boot --name={id} --cloud={cloud}") for id in ids
]
s.run(f"cms cluster build --id={ids} {label}")
print(f"Starting {starting}")
elif arguments.add:
pass
elif arguments.remove:
pass
elif arguments.terminate:
pass
elif arguments.info:
v, label = arguments.verbose or arguments.v or None, arguments.LABEL or None
if label: print(f"Searching for {label}")
virtual_clusters, cloud_clusters = cmdb.collection(
"cluster-virtual"), cmdb.collection("cluster-cloud")
output = {
'virtual': [i for i in virtual_clusters.find(label)],
'cloud': [i for i in cloud_clusters.find(label)]
}
print(output)
return ""
[](https://github.com/TankerHQ/python-{repo}/blob/main/LICENSE)
"""
#
# Find Tests
#
tests = glob.glob('tests/test_**.py')
links = [
"[{name}]({x})".format(x=x, name=os.path.basename(x).replace('.py', '')) for
x in tests]
tests = " * " + "\n * ".join(links)
#
# get manual
#
manual = Shell.run("cms help openapi")
man = []
start = False
for line in manual.splitlines():
start = start or "Usage:" in line
if start:
if not line.startswith("Timer:"):
man.append(line)
manual = textwrap.dedent('\n'.join(man)).strip()
manual = "```bash\n" + manual + "\n```\n"
#
# create readme
#
source = readfile("README-source.md")
readme = source.format(**locals())
def create_file(self, location, content):
Shell.mkdir(os.dirname(path_expand(location)))
writefile(location, content)
def __init__(self):
self.update()
self.IP = Shell.run("minikube ip").strip()
def do_admin(self, args, arguments):
"""
::
Usage:
admin mongo install [--brew] [--download=PATH]
admin mongo create
admin mongo status
admin mongo stats
admin mongo version
admin mongo start
admin mongo stop
admin mongo backup FILENAME
admin mongo load FILENAME
admin mongo security
admin mongo password PASSWORD
admin mongo list
admin rest status
admin rest start
admin rest stop
admin status
admin system info
admin yaml cat
admin yaml check
The admin command performs some administrative functions, such as installing packages, software and services.
It also is used to start services and configure them.
Arguments:
FILENAME the filename for backups
Options:
-f specify the file
Description:
Mongo DB
MongoDB is managed through a number of commands.
The configuration is read from ~/.cloudmesh/cloudmesh4.yaml
First, you need to create a MongoDB database with
cms admin mongo create
Second, you need to start it with
cms admin mongo start
Now you can interact with it to find out the status, the stats,
and the database listing with the commands
cms admin mongo status
cms admin mongo stats
cms admin mongo list
To stop it from running use the command
cms admin mongo stop
System information about your machine can be returned by
cms admin system info
This can be very useful in case you are filing an issue or bug.
"""
# arguments.PATH = arguments['--download'] or None
result = None
if arguments.mongo:
if arguments.install:
print("MongoDB install")
print(79 * "=")
installer = MongoInstaller()
r = installer.install()
return r
elif arguments.status:
mongo = MongoDBController()
state = mongo.status()
if "error" in state["status"]:
Console.error(state["message"])
print(Printer.attribute(state))
else:
data = dotdict()
for pid in state['output']:
entry = state['output'][pid]
data["pid"] = state['output'][pid]
data["command"] = state['output'][pid][
'command'].strip()
print(Printer.dict(data, order=["pid", "command"]))
Console.ok(str(data.pid['pid']) + " " + state["message"])
elif arguments.version:
print("MongoDB Version")
print(79 * "=")
mongo = MongoDBController()
r = mongo.version()
print(r)
elif arguments.security:
mongo = MongoDBController()
mongo.set_auth()
print()
elif arguments.create:
print("MongoDB create")
MongoDBController().create()
elif arguments.start:
print("MongoDB start")
MongoDBController().start(security=True)
elif arguments.stop:
print("MongoDB stop")
MongoDBController().stop()
elif arguments.backup:
print("MongoDB backup")
MongoDBController().dump(arguments.get('FILENAME'))
elif arguments.load:
print("MongoDB backup")
MongoDBController().restore(arguments.get('FILENAME'))
elif arguments.stats:
mongo = MongoDBController()
r = mongo.stats()
if len(r) > 0:
print(Printer.attribute(r))
Console.ok("ok")
else:
Console.ok("is your MongoDB server running")
elif arguments.list:
mongo = MongoDBController()
r = mongo.list()
if len(r) > 0:
print(
Printer.dict(r, order=["name", "sizeOnDisk", "empty"]))
Console.ok("ok")
else:
Console.ok("is your MongoDB server running")
elif arguments.yaml and arguments.cat:
path = path_expand("~/.cloudmesh/cloudmesh4.yaml")
secrets = [
"AZURE_SUBSCRIPTION_ID", "AZURE_TENANTID", "EC2_ACCESS_ID",
"EC2_SECRET_KEY", "OS_PASSWORD", "MONGO_PASSWORD"
]
with open(path) as f:
content = f.read().split("\n")
for line in content:
if "TBD" not in line:
for attribute in secrets:
if attribute + ":" in line:
line = line.split(":")[0] + ": ********"
break
print(line)
return ""
elif arguments.yaml and arguments.check:
path = path_expand("~/.cloudmesh/cloudmesh4.yaml")
print()
r = Shell.live('/Users/grey/.pyenv/shims/yamllint ' + path)
print(70 * '-')
print(" line:column description")
print()
elif arguments.rest:
if arguments.start:
print("Rest Service start")
raise NotImplementedError
elif arguments.stop:
print("Rest Service stop")
raise NotImplementedError
elif arguments.status:
print("Rest Service status")
raise NotImplementedError
elif arguments.status:
config = Config()
data = config.data["cloudmesh"]["data"]["mongo"]
# self.expanduser()
print("Rest Service status")
print("MongoDB status")
mongo = MongoDBController()
print(mongo)
# mongo.expanduser()
# data = mongo.data
# print ("DDD", data)
# data["MONGO_VERSION"] = '.'.join(str(x) for x in mongo.version())
# print (data)
# print(Printer.attribute(data))
# mongo.set_auth()
elif arguments.system:
s = OperatingSystem.get()
print(Printer.attribute(s))
return result
def do_config(self, args, arguments):
"""
::
Usage:
config -h | --help
config cat [less]
config check
config secinit
config security add (--secret=REGEXP | --exception=REGEXP )
config security rmv (--secret=REGEXP | --exception=REGEXP )
config security list
config encrypt
config decrypt [--nopass]
config edit [ATTRIBUTE]
config set ATTRIBUTE=VALUE
config get ATTRIBUTE [--output=OUTPUT]
config value ATTRIBUTE
config cloud verify NAME [KIND]
config cloud edit [NAME] [KIND]
config cloud list NAME [KIND] [--secrets]
Arguments:
SOURCE the file to encrypted or decrypted.
an .enc is added to the filename or removed form it
dependent of if you encrypt or decrypt
ATTRIBUTE=VALUE sets the attribute with . notation in the
configuration file.
ATTRIBUTE reads the attribute from the container and sets it
in the configuration file
If the attribute is a password, * is written instead
of the character included
REGEXP python regular expression
Options:
--secret=REGEXP ensures all attributes within cloudmesh.yaml
whose dot path matches REGEXP are not encrypted
(even if listed in secrets)
--exception=REGEXP ensures attributes within cloudmesh.yaml whose
dot path matches REGEXP are encrypted
--name=KEYNAME The name of a key
--nopass Indicates if private key is password protected
--output=OUTPUT The output format [default: yaml]
--secrets Print the secrets. Use carefully.
Description:
config check
checks if the ssh key ~/.ssh/id_rsa has a password. Verifies it
through entering the passphrase
Key generation
Keys can be generated with
cms key gen (ssh | pem)
Key validity and password can be verified with
cms key verify (ssh | pem)
key verify (ssh | pem) [--filename=FILENAME] [--pub]
ssh-add
cms config encrypt
Encrypts the config data at-rest. This means that the data
is encrypted when not in use. This command is reliant upon
the cloudmesh.security.secrets attribute and the
cloudmesh.security.exceptions attribute within the
cloudmesh.yaml file. Note, that the encrypted data is not
encrypted upon request/query to the attribute. This means
you must decrypt the config when needed in use and
re-encrypt when not using the file, or delivering the file.
1. cloudmesh.security.secrets:
This attribute will hold a list of python regular
expressions that detail which attributes will be
encrypted by the command.
ex) .*: will encrypt all attributes
ex) .*mdbpwd.*: will encrypt all paths with mdbpwd
2. cloudmesh.security.exceptions:
This attribute will hold a list of python regular
expressions that detail which attributes will not
be encrypted by the command.
ex) .*pubkey.*: ensures no pubkey path is encrypted
security add --secret=REGEXP
Adds valid REGEXP to the cloudmesh.security.secrets section
security rmv --secret=REGEXP
Removes REGEXP from the cloudmesh.security.secrets section
security add --exception=REGEXP
Adds valid REGEXP to cloudmesh.security.exceptions section
security rmv --exception=REGEXP
Removes REGEXP from cloudmesh.security.exceptions section
security list
Prints a list of all the attribute dot-paths that are
referenced by cms config encryption and decryption commands
cms config decrypt
Decrypts the config data that was held in rest. This
command decrypts and attributes that were encrypted
using the sister `cms config encrypt` command.
config set ATTRIBUTE=VALUE
config set profile.name=Gregor
In case the ATTRIBUTE is the name of a cloud defined under
cloudmesh.cloud, the value will be written into the credentials
attributes for that cloud this way you can safe a lot of
typing. An example is
cms config set aws.AWS_TEST=Gregor
which would write the AWS_TEST attribute in the credentials
of the cloud aws. This can naturally be used to set for
example username and password.
"""
# d = Config() #~/.cloudmesh/cloudmesh.yaml
# d = Config(encryted=True) # ~/.cloudmesh/cloudmesh.yaml.enc
map_parameters(arguments, "exception", "keep", "nopass", "output",
"secret", "secrets")
source = arguments.SOURCE or path_expand("~/.cloudmesh/cloudmesh.yaml")
destination = source + ".enc"
if arguments.cloud and arguments.edit and arguments.NAME is None:
path = path_expand("~/.cloudmesh/cloudmesh.yaml")
print(path)
Shell.edit(path)
return ""
cloud = arguments.NAME
kind = arguments.KIND
if kind is None:
kind = "cloud"
configuration = Config()
if arguments.cloud and arguments.verify:
service = configuration[f"cloudmesh.{kind}.{cloud}"]
result = {"cloudmesh": {"cloud": {cloud: service}}}
action = "verify"
banner(
f"{action} cloudmesh.{kind}.{cloud} in ~/.cloudmesh/cloudmesh.yaml"
)
print(yaml.dump(result))
flat = flatten(service, sep=".")
for attribute in flat:
if "TBD" in str(flat[attribute]):
Console.error(
f"~/.cloudmesh.yaml: Attribute cloudmesh.{cloud}.{attribute} contains TBD"
)
elif arguments.cloud and arguments.list:
service = configuration[f"cloudmesh.{kind}.{cloud}"]
result = {"cloudmesh": {"cloud": {cloud: service}}}
action = "list"
banner(
f"{action} cloudmesh.{kind}.{cloud} in ~/.cloudmesh/cloudmesh.yaml"
)
lines = yaml.dump(result).split("\n")
secrets = not arguments.secrets
result = Config.cat_lines(lines, mask_secrets=secrets)
print(result)
elif arguments.cloud and arguments.edit:
#
# there is a duplicated code in config.py for this
#
action = "edit"
banner(
f"{action} cloudmesh.{kind}.{cloud}.credentials in ~/.cloudmesh/cloudmesh.yaml"
)
credentials = configuration[
f"cloudmesh.{kind}.{cloud}.credentials"]
print(yaml.dump(credentials))
for attribute in credentials:
if "TBD" in credentials[str(attribute)]:
value = credentials[attribute]
result = input(f"Please enter {attribute}[{value}]: ")
credentials[attribute] = result
# configuration[f"cloudmesh.{kind}.{cloud}.credentials"] = credentials
print(
yaml.dump(
configuration[f"cloudmesh.{kind}.{cloud}.credentials"]))
elif arguments["edit"] and arguments["ATTRIBUTE"]:
attribute = arguments.ATTRIBUTE
config = Config()
config.edit(attribute)
config.save()
return ""
elif arguments.cat:
content = Config.cat()
import shutil
columns, rows = shutil.get_terminal_size(fallback=(80, 24))
lines = content.split("\n")
counter = 1
for line in lines:
if arguments.less:
if counter % (rows - 2) == 0:
x = input().split("\n")[0].strip()
if x != '' and x in 'qQxX':
return ""
print(line)
counter += 1
return ""
elif arguments.check:
Config.check()
elif arguments.encrypt:
config = Config()
config.encrypt()
elif arguments.decrypt:
config = Config()
config.decrypt(ask_pass=not arguments.nopass)
elif arguments.set:
config = Config()
clouds = config["cloudmesh.cloud"].keys()
line = arguments["ATTRIBUTE=VALUE"]
attribute, value = line.split("=", 1)
cloud, field = attribute.split(".", 1)
if cloud in clouds:
attribute = f"cloudmesh.cloud.{cloud}.credentials.{field}"
elif not attribute.startswith("cloudmesh."):
attribute = f"cloudmesh.{attribute}"
config[attribute] = value
config.save()
elif arguments.value:
config = Config()
attribute = arguments.ATTRIBUTE
if not attribute.startswith("cloudmesh."):
attribute = f"cloudmesh.{attribute}"
try:
value = config[attribute]
if type(value) == dict:
raise Console.error("the variable is a dict")
else:
print(f"{value}")
except Exception as e:
print(e)
return ""
elif arguments.secinit:
config = Config()
secpath = path_expand(config['cloudmesh.security.secpath'])
if not os.path.isdir(secpath):
Shell.mkdir(secpath) # Use Shell that makes all dirs as needed
elif arguments.security and arguments.list:
config = Config()
secrets = config.get_list_secrets()
for s in secrets:
Console.msg(s)
elif arguments.security:
# Get the regular expression from command line
regexp = None
if arguments.secret:
regexp = arguments.secret
elif arguments.exception:
regexp = arguments.exception
# Verify argument is valid python regular expression
try:
r = re.compile(regexp)
except re.error:
Console.error(f"Invalid Python RegExp:{regexp}")
sys.exit()
config = Config()
path = None
section = None
# Assign information based on arguments
if arguments.secret:
path = 'cloudmesh.security.secrets'
section = "secrets"
elif arguments.exception:
path = 'cloudmesh.security.exceptions'
section = "exceptions"
# Get current list of regular expressions from related section
exps = config[path]
# Add argument to expressions in related section
if arguments.add:
if regexp not in exps:
config[path].append(regexp)
config.save()
Console.ok(f"Added {regexp} to {section}")
else:
Console.warning(f"{regexp} already in {section}")
# Remove argument from expressions in related section
elif arguments.rmv:
if regexp in exps:
config[path].remove(regexp)
config.save()
Console.ok(f"Removed {regexp} from {section}")
else:
Console.warning(f"{regexp} not in {section}")
elif arguments.get:
print()
config = Config()
clouds = config["cloudmesh.cloud"].keys()
attribute = arguments.ATTRIBUTE
try:
cloud, field = attribute.split(".", 1)
field = f".{field}"
except:
cloud = attribute
field = ""
if cloud in clouds:
attribute = f"cloudmesh.cloud.{cloud}{field}"
elif not attribute.startswith("cloudmesh."):
attribute = f"cloudmesh.{attribute}"
try:
value = config[attribute]
if type(value) == dict:
print(Printer.write(value, output=arguments.output))
else:
print(f"{attribute}={value}")
except Exception as e:
print(e)
return ""
return ""
def do_var(self, args, arguments):
"""
::
Usage:
var list
var clear
var delete NAME
var NAME=VALUE
var NAME
var
Arguments:
NAME the name of the variable
VALUE the value of the variable
FILENAME the filename of the variable
Description:
Manage persistent variables
var NAME=VALUE
sets the variable with the name to the value
if the value is one of data, time, now it will be
replaced with the value at this time, the format will be
date 2017-04-14
time 11:30:33
now 2017-04-14 11:30:41
It will wbe replaced accordingly
The value can also refer to another variable name.
In this case the current value will be copied in the named
variable. As we use the $ sign it is important to distinguish
shell variables from cms variables while using proper quoting.
Examples include:
cms var a=\$b
cms var 'a=$b'
cms var a=val.b
The previous command copy the value from b to a. The val command
was added to avoid quoting.
""" # noqa: W605
if args == '':
arguments["list"] = True
variables = Variables()
if arguments["NAME=VALUE"]:
if '=' in arguments["NAME=VALUE"]:
name, value = arguments["NAME=VALUE"].split("=", 1)
if name in ['debug', 'trace', 'timer']:
value = str(str(value).lower() in ['1', 'on', 'true'])
else:
name = arguments["NAME=VALUE"]
try:
value = variables[name]
except:
value = None
if arguments["clear"]:
variables.clear()
elif arguments["list"]:
for name in variables:
value = variables[name]
print(name, "=", "'", value, "'", sep="")
elif arguments.delete:
del variables[arguments.NAME]
elif name and not value:
Console.error("variable {name} does not exist".format(name=name))
elif name and not value:
print(variables[arguments.NAME])
elif name and value:
if name in ["dryrun"]:
if value.lower() in ["on", "true", "1", "t"]:
value = True
elif value.lower() in ["off", "false", "0", "f"]:
value = True
else:
Console.error(f"value of {name} must be True/False")
return ""
elif value.startswith("cloudmesh."):
try:
from cloudmesh.configuration.Config import Config
config = Config()
except:
Console.error("cloudmesh configuration not loaded.")
try:
value = config[value]
except:
Console.error(f"problem reading {value}")
return ""
elif value.startswith("!"):
# cms set a=\!pwd
command = value[1:]
value = Shell.run(command)
elif value.startswith("py "):
# cms set a=\!pwd
command = value.split("py ")[1]
value = eval(command)
elif value == "time":
value = datetime.now().strftime("%H:%M:%S")
elif value == "date":
value = datetime.now().strftime("%Y-%m-%d")
elif value == "now":
value = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
elif value.startswith("value."):
var = value.replace("value.", "")
value = variables[var]
elif value.startswith("$"):
var = value.replace("$", "")
value = variables[var]
print(name, "=", "'", value, "'", sep="")
variables[name] = value
# fa19-516-156 E.Cloudmesh.Common.4
# Develop a program that demonstartes the use of cloudmesh.common.Shell.
from cloudmesh.common.Shell import Shell
##Check Python Version
result = Shell.check_python()
print(result)
## Check Valid Shell command
result = Shell.command_exists("print")
print(result)
result = Shell.command_exists("output")
print(result)
result = Shell.terminal_type()
print(result)
def tree(self, directory=None):
source = self._dirname(directory)
r = Shell.execute(f"tree {source}")
print(r)
def do_version(self, args, arguments):
"""
::
Usage:
version pip [PACKAGE]
version [--format=FORMAT] [--check=CHECK] [--number]
Options:
--format=FORMAT the format to print the versions in [default: table]
--check=CHECK boolean tp conduct an additional check [default: True]
Description:
version
Prints out the version number
version pip
Prints the contents of pip list
Limitations:
Package names must not have a . in them instead you need to use -
Thus to query for cloudmesh-cmd5 use
cms version pip cloudmesh-cmd5
"""
# print (arguments)
# print (">", args, "<")
if arguments["pip"]:
# noinspection PyBroadException
try:
package = arguments["PACKAGE"]
if package is None:
result = Shell.execute('pip', ['list', '--format=columns'],
traceflag=False,
witherror=False)
print(result)
else:
if "." in package:
package = package.replace(".", "-")
result = Shell.execute('pip', ['show', package],
traceflag=False,
witherror=False)
print(result)
except Exception as e: # noqa: F841
result = 'N/A'
return ""
python_version, pip_version = Shell.get_python()
# noinspection PyBroadException
try:
git_hash_version = Shell.execute('git',
'log -1 --format=%h',
traceflag=False,
witherror=False)
except: # noqa: E722
git_hash_version = 'N/A'
versions = {
"python": {
"name": "python",
"version": str(python_version)
},
"pip": {
"name": "pip",
"version": str(pip_version)
},
"git": {
"name": "git hash",
"version": str(git_hash_version)
}
}
# dynamically check all installed cloudmesh packages and versions
pipcheck = subprocess.Popen(('pip', 'freeze'), stdout=subprocess.PIPE)
try:
# python 3 returns byte sequence so the decode is necessary
output = subprocess.check_output(
('grep', "cloudmesh"), stdin=pipcheck.stdout).decode("utf-8")
pkglines = output.strip().splitlines()
for pkgline in pkglines:
# print (pkgline)
if "==" in pkgline:
values = pkgline.split("==")
pkg = values[0]
version = values[1].strip()
if pkg != 'cloudmesh-installer':
pname = pkg.replace("cloudmesh-", "cloudmesh.")
i = importlib.import_module(pname)
location = i.__file__
try:
vlocation = path_expand(
os.path.join(os.path.dirname(location),
"__version__.py"))
v = readfile(vlocation).split('"')[1].strip()
except: # noqa: E722
v = "not found"
versions[pkg] = {
"name": pkg,
"package": pname,
"version": version,
"source": location,
"VERSION": v
}
elif "git" in pkgline and ("installer" not in pkgline) and (
"pi_burn" not in pkgline):
pkgline = pkgline.replace(
"-e [email protected]:cloudmesh-community/", "")
pkgline = pkgline.replace(
"-e git+https://github.com/cloudmesh/", "")
pkgline = pkgline.replace("egg=", "")
version, pkg = pkgline.split("#")
pname = pkg.replace("cloudmesh-", "cloudmesh.")
#
# The next line needed to be added as for some reason the is an _ here
#
pname = pkg.replace("cloudmesh_", "cloudmesh.")
try:
i = importlib.import_module(pname)
except: # noqa: E722
continue
location = i.__file__
try:
vlocation = path_expand(
os.path.join(os.path.dirname(location),
"__version__.py"))
v = readfile(vlocation).split('"')[1].strip()
except: # noqa: E722
v = "not found"
versions[pkg] = {
"name": pkg,
"package": pname,
"version": version,
"source": location,
"VERSION": v
}
except subprocess.CalledProcessError as e: # noqa: F841
pass
pipcheck.wait()
# installedpkgs = []
#
# for a preset set of named packages
'''
#pkgs = ['cloudmesh-common', 'cloudmesh-cmd5', 'cloudmesh.comet']
for package in pkgs:
# check version from pip
pipcheck = subprocess.Popen(('pip', 'freeze'), stdout=subprocess.PIPE)
try:
output = subprocess.check_output(('grep', package), stdin=pipcheck.stdout)
version = output.split("==")[1].strip()
versions[package] = {"name": package,
"version": version
}
except subprocess.CalledProcessError as e:
pass
pipcheck.wait()
'''
# __version__ not maintained in package file so this won't work
'''
try:
print ("trying package |%s|" % package)
try_module = __import__(package)
print ("added one package into the list...")
installedpkgs.append(package)
except ImportError as e:
print ("error importing |%s|" % package)
pass
#print (installedpkgs)
for package in installedpkgs:
versions[package] = {package: {"name": package,
"version": str(package.__version__)
}
}
print (versions)
'''
if arguments["--number"]:
data = []
for key, entry in versions.items():
try:
d = {'name': entry['package'], 'version': entry['VERSION']}
data.append(d)
except:
pass
print(
Printer.write(data,
output=arguments["--format"],
order=["name", "version"],
header=["Package", "Version"],
sort_keys="name"))
return ""
else:
print(
Printer.write(
versions,
output=arguments["--format"],
order=["name", "package", "VERSION", "version", "source"],
sort_keys="name"))
if arguments["--check"] in ["True"]:
Shell.check_python()
##E.Cloudmesh.Shell.1,2,3
# E.Cloudmesh.Shell.1 - Installed cmd5 and command cms.
from cloudmesh.common.Shell import Shell
# E.Cloudmesh.Shell.2 - New command with firstname as command name.
## cloudmesh-pratibha directory has been uploaded:
##https://github.com/cloudmesh-community/fa19-516-152/tree/master/cloudmesh-exercises/cloudmesh-pratibha
shell2_result = Shell.execute('cms', 'pratibha --text HelloWorld')
print(shell2_result)
# E.Cloudmesh.Shell.3 - New command with firstname as command name.
## cloudmesh-shell3 directory has been uploaded:
## https://github.com/cloudmesh-community/fa19-516-152/tree/master/cloudmesh-exercises/cloudmesh-shell3
shell3_result = Shell.execute('cms', 'shell3 --text abcd')
print(shell3_result)
shell3_result1 = Shell.execute('cms', 'shell3 --number 45')
print(shell3_result1)
shell3_result2 = Shell.execute('cms', 'shell3 list')
print(shell3_result2)
def login(self):
r = Shell.execute("az login", shell=True)
r = r.replace("true", "True")
r = eval("[\n" + r.split("[")[1])
return r
def azure_delete():
print("setting cloud to azure")
Shell.execute("cms set cloud=azure",shell=True)
print("deleting austin-vm-azure_a1...")
Shell.execute("cms vm delete austin-vm-azure_a1", shell=True)
