"""
#data = yaml.load(cred)['clouds']['openstack']
config = Config()
data = config['cloudmesh.cloud.chameleon.credentials']
pprint (data)
#with open(r'E:\data\fruits.yaml') as file:
# The FullLoader parameter handles the conversion from YAML
# scalar values to Python the dictionary format
# fruits_list = yaml.load(file, Loader=yaml.FullLoader)
conn = connection.Connection(**data)
banner("Connection")
print (conn)
servers = conn.list_servers()
pprint (servers[0]['hostname'])
#projects = conn.identity.projects()
#
#pprint(projects)
#for project in projects:
# pprint(project)
banner("List Networks")
msg = "Hello World!"
variables = Variables()
print("")
Console.ok(msg)
print("")
Console.error(msg)
print("")
Console.msg(msg)
print("")
Console.error(msg, prefix = True, traceflag = True)
variables['debug'] = True
variables['trace'] = True
variables['verbose'] = 10
print("Using Banner")
banner("Hello World!")
print("Using Heading")
class example(object):
def doit(self):
HEADING()
print("Hello World!")
doit("Hello World!")
print("Verbose")
d = {"sample" : "value"}
VERBOSE(d)
def test_results(self):
HEADING()
# storage = self.p.service
service = self.service
banner(f"Benchmark results for {service} Storage")
StopWatch.benchmark()
def run_script(self, name=None, hosts=None):
banner(name)
results = self.run(script=self.script[name], hosts=hosts, verbose=True)
def do_multipass(self, args, arguments):
"""
::
Usage:
multipass list [--output=OUTPUT] [--dryrun]
multipass images [--output=OUTPUT] [--dryrun]
multipass start NAMES [--output=OUTPUT] [--dryrun]
multipass stop NAMES [--output=OUTPUT] [--dryrun]
multipass reboot NAMES [--output=OUTPUT] [--dryrun]
multipass delete NAMES [--output=OUTPUT][--dryrun]
multipass destroy NAMES [--output=OUTPUT][--dryrun]
multipass shell NAMES [--dryrun]
multipass run COMMAND NAMES [--output=OUTPUT] [--dryrun]
multipass info NAMES [--output=OUTPUT] [--dryrun]
multipass suspend NAMES [--output=OUTPUT] [--dryrun]
multipass resume NAMES [--output=OUTPUT] [--dryrun]
multipass destroy NAMES [--dryrun]
multipass create NAMES [--image=IMAGE]
[--size=SIZE]
[--mem=MEMORY]
[--cpus=CPUS]
[--cloud-init=CLOUDINIT]
[--dryrun]
multipass reboot NAMES [--dryrun]
multipass mount SOURCE DESTINATION [--dryrun]
multipass umount SOURCE [--dryrun]
multipass transfer SOURCE DESTINATION [--dryrun]
multipass set key=VALUE [--dryrun]
multipass get [key] [--dryrun]
Interface to multipass
Options:
--output=OUTPUT the output format [default: table]. Other values are yaml, csv and json.
--image=IMAGE the image name to be used to create a VM.
Arguments:
NAMES the names of the virtual machine
Description:
The NAMES can be a parameterized hostname such as
red[0-1,5] = red0,red1,red5
Commands:
First you can see the supported multipass images with
cms multipass images
Create and launch a new vm using
cms multipass create NAMES
Optionally you can provide image name, size, memory, # of cpus to create an instance.
Start one or multiple multipass vms with
cms multipass start NAMES
Stop one or multiple vms with
cms multipass stop NAMES
Gets all multipass internal key values with
cms multipass get
Gets a specific internal key.
cms multipass get KEY
Known keys
client.gui.autostart
client.primary-name
local.driver
are there more?
Reboot (stop and then start) vms with
cms multipass reboot NAMES
Delete one of multiple vms without purging with
cms multipass delete NAMES
Destory multipass vms (delete and purge) with
cms multipass destroy NAMES
Caution: Once destroyed everything in vm will be deleted and cannot be recovered.
WHEN YOU IMPLEMENT A FUNCTION INCLUDE MINIMAL
DOCUMENTATION HERE
"""
name = arguments.NAME
map_parameters(arguments, "dryrun", "refresh", "cloud", "image",
"size", "mem", "cpus", "output")
# so we can use arguments.cloudinit
arguments["cloudinit"] = arguments["--cloud-init"]
image = arguments.image
variables = Variables()
arguments.output = Parameter.find("output", arguments, variables,
"table")
names = Parameter.expand(arguments.NAMES)
VERBOSE(arguments)
if arguments.list:
if arguments.dryrun:
banner("dryrun list")
else:
provider = Provider()
provider.list()
return ""
elif arguments.images:
if arguments.dryrun:
banner("dryrun images")
else:
provider = Provider()
images = provider.images()
print(
provider.Print(images,
kind='image',
output=arguments.output))
return ""
elif arguments.run:
if arguments.dryrun:
banner("dryrun run")
for name in names:
if arguments.dryrun:
Console.ok(f"run {name} {arguments.COMMAND}")
else:
provider = Provider()
provider.run(name, arguments.COMMAND)
return ""
elif arguments.create:
result = ""
if arguments.dryrun:
banner("create")
for name in names:
if arguments.dryrun:
Console.ok(f"dryrun create {name} {image}")
else:
provider = Provider()
result = provider.create(name, image)
VERBOSE(result)
return result
elif arguments.start:
result = ""
if arguments.dryrun:
banner("start")
for name in names:
if arguments.dryrun:
Console.ok(f"dryrun start {name}")
else:
provider = Provider()
result = provider.start(name)
VERBOSE(result)
return result
elif arguments.stop:
result = ""
if arguments.dryrun:
banner("stop")
for name in names:
if arguments.dryrun:
Console.ok(f"dryrun stop {name}")
else:
provider = Provider(name=name)
result = provider.stop(name)
VERBOSE(result)
return result
elif arguments.delete:
result = ""
if arguments.dryrun:
banner("delete")
for name in names:
if arguments.dryrun:
Console.ok(f"dryrun delete {name}")
else:
provider = Provider()
# Default purge is false. Is this ok?
result = provider.delete(name)
VERBOSE(result)
return result
elif arguments.info:
result = ""
if arguments.dryrun:
banner(f"info {name}")
for name in names:
if arguments.dryrun:
Console.ok(f"dryrun info {name}")
else:
provider = Provider()
# Default purge is false. Is this ok?
result = provider.info(name)
VERBOSE(result)
return result
elif arguments.suspend:
result = ""
if arguments.dryrun:
banner("suspend")
for name in names:
if arguments.dryrun:
Console.ok(f"dryrun suspend {name}")
else:
provider = Provider()
result = provider.suspend(name)
VERBOSE(result)
return result
elif arguments.resume:
result = ""
if arguments.dryrun:
banner("resume")
for name in names:
if arguments.dryrun:
Console.ok(f"dryrun resume {name}")
else:
provider = Provider()
result = provider.resume(name)
VERBOSE(result)
return result
elif arguments.destroy:
result = ""
if arguments.dryrun:
banner("destroy")
for name in names:
if arguments.dryrun:
Console.ok(f"dryrun destroy {name}")
else:
provider = Provider()
result = provider.destroy(name)
VERBOSE(result)
return result
elif arguments.reboot:
result = ""
if arguments.dryrun:
banner("reboot")
for name in names:
if arguments.dryrun:
Console.ok(f"dryrun reboot {name}")
else:
provider = Provider()
result = provider.reboot(name)
VERBOSE(result)
return result
elif arguments.shell:
if len(names) > 1:
Console.error("shell must only have one host")
return ""
name = names[0]
if arguments.dryrun:
banner("dryrun shell {name}")
else:
provider = Provider()
provider.shell()
return ""
elif arguments.info:
if arguments.dryrun:
banner("dryrun info")
else:
provider = Provider()
info = provider.info()
print(
provider.Print(info, kind='info', output=arguments.output))
return ""
elif arguments.mount:
if arguments.dryrun:
banner(
f"dryrun mount {arguments.SOURCE} {arguments.DESTINATION}")
else:
provider = Provider()
provider.mount(arguments.SOURCE, arguments.DESTINATION)
# list the mounts and display as table
return ""
else:
Console.error("Not yet implemented")
return ""
def images(self):
banner("images list")
os.system("multipass find")
def create(self,
name=None,
image=None,
size=None,
location=None,
timeout=360,
key=None,
secgroup=None,
ip=None,
user=None,
public=None,
group=None,
metadata=None,
**kwargs):
"""
creates a named node
:param name: the name of the node
:param image: the image used
:param size: the size of the image
:param timeout: a timeout in seconds that is invoked in case the image
does not boot. The default is set to 3 minutes.
:param kwargs: additional arguments passed along at time of boot
:return: the list with the modified dicts
"""
"""
create one node
"""
if not ip and public:
ip = self.find_available_public_ip()
elif ip is not None:
entry = self.list_public_ips(ip=ip, available=True)
if len(entry) == 0:
Console.error("ip not available")
return None
banner("Create Server")
Console.msg(f" Name: {name}")
Console.msg(f" IP: {ip}")
Console.msg(f" Image: {image}")
Console.msg(f" Size: {size}")
Console.msg(f" Public: {public}")
Console.msg(f" Key: {key}")
Console.msg(f" location:{location}")
Console.msg(f" timeout: {timeout}")
Console.msg(f" secgroup:{secgroup}")
Console.msg(f" group: {group}")
# Validate if there is any VM with same tag name and state other than Terminated.
# If there is any VM, throw error
# ec2_reservations = self.info(name)['Reservations']
#
# if ec2_reservations:
# reservation_instances = None
# for reservation in ec2_reservations:
# reservation_instances = list(
# filter(lambda instance: instance['State']['Name'] != 'terminated', reservation['Instances']))
#
# if reservation_instances:
# Console.error("Tag name already exists, Please use different tag name.")
# return
if secgroup is None:
secgroup = 'default'
if key is None:
raise ValueError("Key must be set. Use cms set key=<key name>")
#
# BUG: the tags seem incomplete
#
if metadata is None:
metadata = []
metadata = [{
'Key': 'cm.image',
'Value': image
}, {
'Key': 'cm.name',
'Value': name
}, {
'Key': 'cm.flavor',
'Value': size
}, {
'Key': 'cm.user',
'Value': self.user
}, {
'Key': 'cm.kind',
'Value': "vm"
}, {
'Key': 'cm.status',
'Value': "BOOTING"
}, {
'Key': 'Name',
'Value': name
}]
# VERBOSE(metadata)
new_ec2_instance = self.ec2_resource.create_instances(
ImageId=image,
InstanceType=size,
MaxCount=1,
MinCount=1,
SecurityGroups=[secgroup],
KeyName=key,
TagSpecifications=[{
'ResourceType': 'instance',
'Tags': metadata
}])
# VERBOSE(new_ec2_instance)
new_ec2_instance = new_ec2_instance[0]
waiter = self.ec2_client.get_waiter('instance_exists')
waiter.wait(Filters=[{
'Name': 'instance-id',
'Values': [new_ec2_instance.instance_id]
}],
WaiterConfig={
'Delay': 20,
'MaxAttempts': timeout / 20
})
print()
Console.ok("Instance created...")
print()
# if IP provided, Attach it to new instance
if ip:
self.attach_public_ip(name, ip)
# x = self.ec2_client.describe_instances(InstanceIds=[new_ec2_instance.instance_id])
# VERBOSE(x)
data = self.info(name=name)
# VERBOSE(data)
data['name'] = name
data['kind'] = 'aws'
data['status'] = new_ec2_instance.state['Name'],
data['created'] = new_ec2_instance.launch_time.strftime(
"%m/%d/%Y, %H:%M:%S") if new_ec2_instance.launch_time else '',
data['updated'] = new_ec2_instance.launch_time.strftime(
"%m/%d/%Y, %H:%M:%S") if new_ec2_instance.launch_time else '',
data['name'] = new_ec2_instance.tags[0][
'Value'] if new_ec2_instance.tags else '',
data['instance_id'] = new_ec2_instance.id,
data['image'] = new_ec2_instance.image_id,
data['key_name'] = key,
Console.msg("Waiting for the Public IP address assignment ...")
while True:
try:
public_ip = \
self.ec2_client.describe_instances(
InstanceIds=[new_ec2_instance.id])['Reservations'][0][
'Instances'] \
[0]['PublicIpAddress'],
break
except KeyError:
time.sleep(0.5)
data['public_ips'] = public_ip[0]
data['private_ips'] = new_ec2_instance.private_ip_address
Console.msg(f" Public IP: {data['public_ips']}")
Console.msg(f" Private IP: {data['private_ips']}")
output = self.update_dict(data, kind="vm")[0]
return output
def list(self):
# list instances
banner("list the launched instance")
os.system("multipass list")
print('\n')
def shell(self):
banner("open shell on running instance")
os.system(f"multipass shell")
print('\n')
def delete(self):
banner(f"delete {self.name}")
# delete the instance
os.system(f"multipass delete {self.name}")
def purge(self):
banner(f"purge")
os.system(f"multipass purge")
print('\n')
def stop(self):
banner(f"stop {self.name}")
os.system(f"multipass stop {self.name}")
print('\n')
def launch(self):
banner(f"launch {self.name}")
os.system(f"multipass launch -n {self.name}")
print('\n')
def setup(self):
banner("setup", c="-")
self.user = Config()["cloudmesh.profile.user"]
self.p = Provider(name="aws")
def do_config(self, args, arguments):
"""
::
Usage:
config -h | --help
config cat [less]
config check
config secinit
config security add (--secret=REGEXP | --exception=REGEXP )
config security rmv (--secret=REGEXP | --exception=REGEXP )
config security list
config encrypt
config decrypt [--nopass]
config edit [ATTRIBUTE]
config set ATTRIBUTE=VALUE
config get ATTRIBUTE [--output=OUTPUT]
config value ATTRIBUTE
config cloud verify NAME [KIND]
config cloud edit [NAME] [KIND]
config cloud list NAME [KIND] [--secrets]
Arguments:
SOURCE the file to encrypted or decrypted.
an .enc is added to the filename or removed form it
dependent of if you encrypt or decrypt
ATTRIBUTE=VALUE sets the attribute with . notation in the
configuration file.
ATTRIBUTE reads the attribute from the container and sets it
in the configuration file
If the attribute is a password, * is written instead
of the character included
REGEXP python regular expression
Options:
--secret=REGEXP ensures all attributes within cloudmesh.yaml
whose dot path matches REGEXP are not encrypted
(even if listed in secrets)
--exception=REGEXP ensures attributes within cloudmesh.yaml whose
dot path matches REGEXP are encrypted
--name=KEYNAME The name of a key
--nopass Indicates if private key is password protected
--output=OUTPUT The output format [default: yaml]
--secrets Print the secrets. Use carefully.
Description:
config check
checks if the ssh key ~/.ssh/id_rsa has a password. Verifies it
through entering the passphrase
Key generation
Keys can be generated with
cms key gen (ssh | pem)
Key validity and password can be verified with
cms key verify (ssh | pem)
key verify (ssh | pem) [--filename=FILENAME] [--pub]
ssh-add
cms config encrypt
Encrypts the config data at-rest. This means that the data
is encrypted when not in use. This command is reliant upon
the cloudmesh.security.secrets attribute and the
cloudmesh.security.exceptions attribute within the
cloudmesh.yaml file. Note, that the encrypted data is not
encrypted upon request/query to the attribute. This means
you must decrypt the config when needed in use and
re-encrypt when not using the file, or delivering the file.
1. cloudmesh.security.secrets:
This attribute will hold a list of python regular
expressions that detail which attributes will be
encrypted by the command.
ex) .*: will encrypt all attributes
ex) .*mdbpwd.*: will encrypt all paths with mdbpwd
2. cloudmesh.security.exceptions:
This attribute will hold a list of python regular
expressions that detail which attributes will not
be encrypted by the command.
ex) .*pubkey.*: ensures no pubkey path is encrypted
security add --secret=REGEXP
Adds valid REGEXP to the cloudmesh.security.secrets section
security rmv --secret=REGEXP
Removes REGEXP from the cloudmesh.security.secrets section
security add --exception=REGEXP
Adds valid REGEXP to cloudmesh.security.exceptions section
security rmv --exception=REGEXP
Removes REGEXP from cloudmesh.security.exceptions section
security list
Prints a list of all the attribute dot-paths that are
referenced by cms config encryption and decryption commands
cms config decrypt
Decrypts the config data that was held in rest. This
command decrypts and attributes that were encrypted
using the sister `cms config encrypt` command.
config set ATTRIBUTE=VALUE
config set profile.name=Gregor
In case the ATTRIBUTE is the name of a cloud defined under
cloudmesh.cloud, the value will be written into the credentials
attributes for that cloud this way you can safe a lot of
typing. An example is
cms config set aws.AWS_TEST=Gregor
which would write the AWS_TEST attribute in the credentials
of the cloud aws. This can naturally be used to set for
example username and password.
"""
# d = Config() #~/.cloudmesh/cloudmesh.yaml
# d = Config(encryted=True) # ~/.cloudmesh/cloudmesh.yaml.enc
map_parameters(arguments, "exception", "keep", "nopass", "output",
"secret", "secrets")
source = arguments.SOURCE or path_expand("~/.cloudmesh/cloudmesh.yaml")
destination = source + ".enc"
if arguments.cloud and arguments.edit and arguments.NAME is None:
path = path_expand("~/.cloudmesh/cloudmesh.yaml")
print(path)
Shell.edit(path)
return ""
cloud = arguments.NAME
kind = arguments.KIND
if kind is None:
kind = "cloud"
configuration = Config()
if arguments.cloud and arguments.verify:
service = configuration[f"cloudmesh.{kind}.{cloud}"]
result = {"cloudmesh": {"cloud": {cloud: service}}}
action = "verify"
banner(
f"{action} cloudmesh.{kind}.{cloud} in ~/.cloudmesh/cloudmesh.yaml"
)
print(yaml.dump(result))
flat = flatten(service, sep=".")
for attribute in flat:
if "TBD" in str(flat[attribute]):
Console.error(
f"~/.cloudmesh.yaml: Attribute cloudmesh.{cloud}.{attribute} contains TBD"
)
elif arguments.cloud and arguments.list:
service = configuration[f"cloudmesh.{kind}.{cloud}"]
result = {"cloudmesh": {"cloud": {cloud: service}}}
action = "list"
banner(
f"{action} cloudmesh.{kind}.{cloud} in ~/.cloudmesh/cloudmesh.yaml"
)
lines = yaml.dump(result).split("\n")
secrets = not arguments.secrets
result = Config.cat_lines(lines, mask_secrets=secrets)
print(result)
elif arguments.cloud and arguments.edit:
#
# there is a duplicated code in config.py for this
#
action = "edit"
banner(
f"{action} cloudmesh.{kind}.{cloud}.credentials in ~/.cloudmesh/cloudmesh.yaml"
)
credentials = configuration[
f"cloudmesh.{kind}.{cloud}.credentials"]
print(yaml.dump(credentials))
for attribute in credentials:
if "TBD" in credentials[str(attribute)]:
value = credentials[attribute]
result = input(f"Please enter {attribute}[{value}]: ")
credentials[attribute] = result
# configuration[f"cloudmesh.{kind}.{cloud}.credentials"] = credentials
print(
yaml.dump(
configuration[f"cloudmesh.{kind}.{cloud}.credentials"]))
elif arguments["edit"] and arguments["ATTRIBUTE"]:
attribute = arguments.ATTRIBUTE
config = Config()
config.edit(attribute)
config.save()
return ""
elif arguments.cat:
content = Config.cat()
import shutil
columns, rows = shutil.get_terminal_size(fallback=(80, 24))
lines = content.split("\n")
counter = 1
for line in lines:
if arguments.less:
if counter % (rows - 2) == 0:
x = input().split("\n")[0].strip()
if x != '' and x in 'qQxX':
return ""
print(line)
counter += 1
return ""
elif arguments.check:
Config.check()
elif arguments.encrypt:
config = Config()
config.encrypt()
elif arguments.decrypt:
config = Config()
config.decrypt(ask_pass=not arguments.nopass)
elif arguments.set:
config = Config()
clouds = config["cloudmesh.cloud"].keys()
line = arguments["ATTRIBUTE=VALUE"]
attribute, value = line.split("=", 1)
cloud, field = attribute.split(".", 1)
if cloud in clouds:
attribute = f"cloudmesh.cloud.{cloud}.credentials.{field}"
elif not attribute.startswith("cloudmesh."):
attribute = f"cloudmesh.{attribute}"
config[attribute] = value
config.save()
elif arguments.value:
config = Config()
attribute = arguments.ATTRIBUTE
if not attribute.startswith("cloudmesh."):
attribute = f"cloudmesh.{attribute}"
try:
value = config[attribute]
if type(value) == dict:
raise Console.error("the variable is a dict")
else:
print(f"{value}")
except Exception as e:
print(e)
return ""
elif arguments.secinit:
config = Config()
secpath = path_expand(config['cloudmesh.security.secpath'])
if not os.path.isdir(secpath):
Shell.mkdir(secpath) # Use Shell that makes all dirs as needed
elif arguments.security and arguments.list:
config = Config()
secrets = config.get_list_secrets()
for s in secrets:
Console.msg(s)
elif arguments.security:
# Get the regular expression from command line
regexp = None
if arguments.secret:
regexp = arguments.secret
elif arguments.exception:
regexp = arguments.exception
# Verify argument is valid python regular expression
try:
r = re.compile(regexp)
except re.error:
Console.error(f"Invalid Python RegExp:{regexp}")
sys.exit()
config = Config()
path = None
section = None
# Assign information based on arguments
if arguments.secret:
path = 'cloudmesh.security.secrets'
section = "secrets"
elif arguments.exception:
path = 'cloudmesh.security.exceptions'
section = "exceptions"
# Get current list of regular expressions from related section
exps = config[path]
# Add argument to expressions in related section
if arguments.add:
if regexp not in exps:
config[path].append(regexp)
config.save()
Console.ok(f"Added {regexp} to {section}")
else:
Console.warning(f"{regexp} already in {section}")
# Remove argument from expressions in related section
elif arguments.rmv:
if regexp in exps:
config[path].remove(regexp)
config.save()
Console.ok(f"Removed {regexp} from {section}")
else:
Console.warning(f"{regexp} not in {section}")
elif arguments.get:
print()
config = Config()
clouds = config["cloudmesh.cloud"].keys()
attribute = arguments.ATTRIBUTE
try:
cloud, field = attribute.split(".", 1)
field = f".{field}"
except:
cloud = attribute
field = ""
if cloud in clouds:
attribute = f"cloudmesh.cloud.{cloud}{field}"
elif not attribute.startswith("cloudmesh."):
attribute = f"cloudmesh.{attribute}"
try:
value = config[attribute]
if type(value) == dict:
print(Printer.write(value, output=arguments.output))
else:
print(f"{attribute}={value}")
except Exception as e:
print(e)
return ""
return ""
def find(self):
banner(f"find available multipass images")
os.system(f"multipass find")
print('\n')
def list(self):
# list instances
banner("list")
os.system("multipass ls")
def create(self,
name=None,
image=None,
size=None,
location=None,
timeout=360,
key=None,
secgroup=None,
ip=None,
user=None,
public=True,
group=None,
metadata=None,
cloud=None,
**kwargs):
"""
creates a named node
:param group: the list of groups the vm belongs to
:param name: the name of the node
:param image: the image used
:param size: the size of the image
:param timeout: a timeout in seconds that is invoked in case the image
does not boot. The default is set to 3 minutes.
:param kwargs: additional arguments HEADING(c=".")ed along at time of
boot
:return:
"""
image_use = None
flavor_use = None
# keyname = Config()["cloudmesh"]["profile"]["user"]
# ex_keyname has to be the registered keypair name in cloud
"""
https://docs.openstack.org/openstacksdk/latest/user/connection.html#openstack.connection.Connection.create_server
"""
if 'flavor' in kwargs and size is None:
size = kwargs['flavor']
# Guess user name
if user is None:
user = Image.guess_username(image)
# image_name = image.lower()
# if image_name.startswith("cc-"):
# user = "******"
# if "centos" in image_name:
# user = "******"
# elif "ubuntu" in image_name:
# user = "******"
# get IP
if not ip and public:
ip = self.find_available_public_ip()
# pprint(entry)
elif ip is not None:
entry = self.list_public_ips(ip=ip, available=True)
if len(entry) == 0:
print("ip not available")
raise ValueError(f"The ip can not be assigned {ip}")
if type(group) == str:
groups = Parameter.expand(group)
banner("Create Server")
print(" Name: ", name)
print(" User: "******" IP: ", ip)
print(" Image: ", image)
print(" Size: ", size)
print(" Public: ", public)
print(" Key: ", key)
print(" location:", location)
print(" timeout: ", timeout)
print(" secgroup:", secgroup)
print(" group: ", group)
print(" groups: ", groups)
print()
try:
server = self.cloudman.create_server(name,
flavor=size,
image=image,
key_name=key,
security_groups=[secgroup],
timeout=timeout
# tags=groups,
# wait=True
)
server['user'] = user
r = self.cloudman.wait_for_server(server)
s = self.cloudman.add_ips_to_server(server, ips=ip)
variables = Variables()
variables['vm'] = name
if metadata is None:
metadata = {}
metadata['image'] = image
metadata['flavor'] = size
self.cloudman.set_server_metadata(server, metadata)
self.add_secgroup(name=secgroup)
# server = self.cloudman.compute.wait_for_server(server)
# print("ssh -i {key} root@{ip}".format(
# key=PRIVATE_KEYPAIR_FILE,
# ip=server.access_ipv4))
except Exception as e:
Console.error("Problem starting vm", traceflag=True)
print(e)
raise RuntimeError
return self.update_dict(server, kind="vm")[0]
# sp20-516-255 E.Cloudmesh.Common.1
# Program that demonstrates the use of banner, HEADING, and VERBOSE.
from cloudmesh.common.util import banner
from cloudmesh.common.util import HEADING
from cloudmesh.common.debug import VERBOSE
bannerText = "Banner Test"
## banner
banner(bannerText, "*", color="BLUE")
## HEADING
headingText = "Heading Test"
HEADING(headingText)
#VERBOSE
verboseText = {"Banner": bannerText, "Heading": headingText}
VERBOSE(verboseText)
def __init__(self, service=None, config="~/.cloudmesh/cloudmesh.yaml",
**kwargs):
super().__init__(service=service, config=config)
if kwargs.get("debug"):
print("Inside init of local provider")
print(self.kind)
print(kwargs.get('sourceObj'))
print(kwargs.get('target'))
print(kwargs.get('targetObj'))
print(self.credentials)
# Processing --source/service and --target arguments separately.
# This is a provider class for local storage hence --source/service will \
# always be "local"
self.sourceCSP = self.service
try:
self.config = Config(config_path=config)
self.yaml_content_source = self.config["cloudmesh.storage."
f"{self.sourceCSP}"]
self.source_kind = self.yaml_content_source["cm"]["kind"]
self.source_credentials = self.yaml_content_source["credentials"]
print("Accessing local storage location.")
if kwargs.get('sourceObj'):
self.local_location = Path(self.yaml_content_source['default'][
'directory'],
kwargs.get('sourceObj'))
else:
self.local_location = self.yaml_content_source['default'][
'directory']
if kwargs.get("debug"):
print(f"\nLocal location to access {self.local_location}")
if kwargs.get('target'):
self.targetCSP = kwargs.get('target')
self.yaml_content_target = self.config["cloudmesh.storage."
f"{self.targetCSP}"]
self.target_kind = self.yaml_content_target["cm"]["kind"]
self.target_credentials = self.yaml_content_target[
"credentials"]
self.target_container = self.target_credentials["container"]
except Exception as e:
Console.error(f"Couldn't access cloudmesh.yaml. Error - {e}")
return ()
if kwargs.get("debug"):
VERBOSE(self.yaml_content_source)
if kwargs.get('target'):
VERBOSE(self.yaml_content_target)
banner(f"Source CSP: {self.source_kind}")
if kwargs.get('target'):
banner(f"Target CSP: {self.target_kind}")
# Creating connection with the target CSP. This done only if the
# --target argument is provided. Only "copy" command is expected to
# have --target argument.
if kwargs.get('target'):
if self.target_kind == "azureblob":
print("Create Azure connection.")
if 'TBD' == self.target_credentials["access_key_id"] \
or 'TBD' == self.target_credentials["secret_access_key"] \
or 'TBD' == self.target_credentials["region"]:
Console.error("Critical details missing from .yaml file. "
"TBD not allowed. Please check.")
try:
self.s3_client = boto3.client(
's3',
aws_access_key_id=self.target_credentials[
"access_key_id"],
aws_secret_access_key=self.target_credentials[
"secret_access_key"],
region_name=self.target_credentials["region"]
)
Console.ok(
f"Successful connection to {self.target_kind} is "
f"made.")
except ClientError as e:
Console.error(e, prefix=True, traceflag=True)
elif self.kind == "gcpbucket":
print("Create GCP connection.")
raise NotImplementedError
else:
raise NotImplementedError
def do_register(self, args, arguments):
"""
::
Usage:
register aws yaml
register aws [FILENAME] [--keep]
register azure [FILENAME] [--keep]
register google [FILENAME] [--keep]
register chameleon [FILENAME] [--keep]
register new [-v] KIND SERVICE NAME [ATTRIBUTES...]
register list KIND SERVICE NAME [ATTRIBUTES...]
This command adds the registration information the th cloudmesh
yaml file. The permissions of theFILENAME will also be changed.
A y/n question will be asked if the files with the filename should
be deleted after integration
Arguments:
FILENAME a filename in which the cloud credentials are stored
Options:
--keep keeps the file with the filename.
Description:
AWS
AWS dependent on how you downloaded the credentials will either
use the filename `credentials.csv` or `accessKey.csv`
Our command is smart provides some convenience functionality.
1. If either file is found in `~/Downloads`, it is moved to
`~/.cloudmesh` and the permissions are changed
2. If such a file already exists there it will ask if it should
be overwritten in case the content is not the same
3. The content of the file will be read to determine if it is
likely to be an AWS credential
4. The credential will be added to the cloudmesh yaml file
Azure (TODO: THIS HAS TO BE IMPLEMENTED)
To use this command you must have an Azure account and Azure CLI
installed. Mote deatils to this can be found at:
TBD: include link
This command leverages the Azure CLI to extract specific account
parameters and update the `cloudmesh.yaml` configuration file.
1. When this command is run, the `az login` Azure CLI command is
executed, prompting you to login to your azure account via
your default browser, which should then redirect you back to
your CLI.
2. The azure cli command for `az account show` is then
referenced to pull the account's subscription id and tenant
id.
3. Run the command
`az ad sp create-for-rbac --name http://cloudmesh`
to reference the application id and secret key.
NOTE: If there is already a secret key reference in the
`cloudmesh.yaml` file, it will be rendered useless once the new
secret key is obtained; requiring you to ensure the key has
been successfully applied in the next step to the
configuration file and cms init is called.
4. The credentials will be added to the
`~/.cloudmesh.yaml` configuration file.
Google
Is not yet implemented
Chameleon Cloud
is not yet implemented
Register from template:
register list KIND SERVICE NAME [ATTRIBUTES...]
This command lists the template for a provider and fills it out
with the attributes. It can be used to check what information
would be added with the new command. The template is included in
the Provider itself. Leaving the attributes off, just prints the
template
Examples:
cms register list compute openstack ABC
cms register list compute openstack ABC uri=TBD tenant=TBD region=TBD
In the last example the values for uri, tennant, and region
will be cahnged to TBD
register new KIND SERVICE NAME ATTRIBUTES...
A new cloud can be added to the existing cloudmesh.yaml file
with thsi command. AN example is
cms register -v compute aws ABC \
region=one \
EC2_ACCESS_ID=TBD
EC2_SECRET_KEY=TBD
EC2_SECRET_KEY=TBD
This command can also be used to overwrite an existing entry
"""
if arguments.aws and arguments.yaml:
AWSReg = importlib.import_module("cloudmesh.register.AWSRegister")
AWSregisterer = AWSReg.AWSRegister()
AWSregisterer.register()
elif arguments.azure:
if arguments.yaml:
Registry = importlib.import_module(
"cloudmesh.register.AzRegister")
AZregisterer = Registry.AzRegister()
AZregisterer.register()
return ""
elif arguments.google:
from cloudmesh.google.storage.Provider import Provider
banner("Read the specification from json and write to yaml file")
path = path_expand(arguments.FILE_JSON
or "~/.cloudmesh/google.json")
name = arguments.storage or "google"
Provider.json_to_yaml(name, filename=path)
Console.error("deleting {path} not implemented")
return ""
elif arguments.chameleon:
Console.error("not yet implemented")
return ""
elif arguments.list:
kind = arguments.KIND
service = arguments.SERVICE
attributes = arguments.ATTRIBUTES
name = arguments.NAME
replacements = {'name': name, 'service': service}
for attribute in attributes:
key, value = attribute.split("=")
replacements[key] = value
if kind == "compute" and service == "openstack":
from cloudmesh.compute.openstack.Provider import Provider
elif kind == "compute" and service == "azure":
from cloudmesh.compute.azure.Provider import Provider
elif kind == "compute" and service == "aws":
from cloudmesh.compute.aws.Provider import Provider
elif kind == "compute" and service == "oracle":
from cloudmesh.oracle.compute.Provider import Provider
elif kind == "storage" and service == "google":
from cloudmesh.google.storage.Provider import Provider
sample = Provider.sample
try:
if len(attributes) > 0:
sample = sample.format(**replacements)
except KeyError as e:
Console.error(f"Value for {e} is not specified")
print(dedent(sample))
elif arguments.new:
kind = arguments.KIND
service = arguments.SERVICE
attributes = arguments.ATTRIBUTES
name = arguments.NAME
replacements = {'name': name, 'service': service}
for attribute in attributes:
key, value = attribute.split("=")
replacements[key] = value
if kind == "compute" and service == "openstack":
from cloudmesh.compute.openstack.Provider import Provider
elif kind == "compute" and service == "azure":
from cloudmesh.compute.azure.Provider import Provider
elif kind == "compute" and service == "aws":
from cloudmesh.compute.aws.Provider import Provider
elif kind == "compute" and service == "oracle":
from cloudmesh.oracle.compute.Provider import Provider
elif kind == "storage" and service == "google":
from cloudmesh.google.storage.Provider import Provider
else:
Console.error("The provider {google} for {kind}"
" does not have a sample configuration")
return ""
if kind in ["compute", "cloud"]:
kind = "cloud"
if kind in ["storage"]:
kind = "storage"
else:
Console.error("the kind {kind} is not supported")
return ""
sample = Provider.sample
try:
sample = sample.format(**replacements)
if arguments["-v"]:
print(sample)
Entry.add(entry=sample,
base=f"cloudmesh.{kind}",
path="~/.cloudmesh/cloudmesh.yaml")
except KeyError as e:
Console.error(f"Value for {e} is not specified")
print(dedent(sample))
return ""
def setup(self):
self.i = Inventory()
banner("Info")
self.i.info()
def update_slaves(self,hosts):
banner("Updating $SPARK_HOME/conf/slaves file")
command5 = f"echo 'pi@{hosts}' >> $SPARK_HOME/conf/slaves "
print(command5)
os.system(command5)
def showBanner(self, msg):
banner(msg)
# fa20-516-238 E.Cloudmesh.Common.3
from cloudmesh.common.util import banner
from cloudmesh.common.FlatDict import FlatDict
class CloudmeshCommon:
def demo_flatdict(self):
test_dict = {
"country": "UNITED STATES OF AMERICA",
"attributes": {
"Language": "English",
"Capital": "Washington D.C.",
"Currency": "USD"
}
}
print(test_dict)
flat_list = FlatDict(test_dict, sep=".")
banner("Flattened list for test_dict")
print(flat_list)
if __name__ == '__main__':
demo = CloudmeshCommon()
banner("FlatDict Demo")
demo.demo_flatdict()
# sp20-516-239 E.Cloudmesh.Common.1
from cloudmesh.common.util import banner
banner("This is my banner")
from cloudmesh.common.util import HEADING
HEADING("Spring 2020 Cloud Computing Class")
print("Demonstration of heading")
from cloudmesh.common.debug import VERBOSE
m = {"first_name": "Sara"}
VERBOSE(m)
def test_banner(self):
banner("START", color="RED")
# fa20-516-238 E.Cloudmesh.Common.2
from cloudmesh.common.util import banner
from cloudmesh.common.dotdict import dotdict
class CloudmeshCommon:
def demo_dotdict(self):
student = {"name": "Ishan Mishra", "id": "238", "subject": "ECC"}
student = dotdict(student)
print("Student Name:", student.name, "\n")
print("Student ID:", student.id, "\n")
print("Subject Enrolled:", student.subject, "\n")
if __name__ == '__main__':
demo = CloudmeshCommon()
banner("\t\t Student's Details", c="x", prefix="@")
demo.demo_dotdict()
from cloudmesh.common.util import banner, HEADING
from cloudmesh.common.debug import VERBOSE
banner("This is my first print command",
debug=True,
label="Label for Banner",
color="GREEN")
banner("Trying out Heading")
HEADING("This is interesting!", c='#', color='HEADER')
banner("This is Verbose")
d = {
"January": "1",
"February": "2",
"March": "3",
"April": "4",
"May": "5",
"June": "6",
"July": "7",
"August": "8",
"September": "9",
"October": "10",
"November": "11",
"December": '12'
}
VERBOSE(d, label="Months", color='RED', verbose=-10)
def add(self, n1, n2):
HEADING()
result = n1 + n2
return result
def divide(self, n1, n2):
HEADING()
result = 0
try:
result = n1 / n2
except ZeroDivisionError as err:
self.operErrors.append({
"operation":
"divide",
"Error":
f"Error when dividing {n1} by {n2} : {err}"
})
return result
if __name__ == '__main__':
mathUtil = MathUtil()
banner("Start:: Testing Math Util")
sum = mathUtil.add(3, 5)
print(sum)
div = mathUtil.divide(50, 0)
banner("End:: Testing Math Util")
VERBOSE(mathUtil.operErrors[0])
