""" #data = yaml.load(cred)['clouds']['openstack'] config = Config() data = config['cloudmesh.cloud.chameleon.credentials'] pprint (data) #with open(r'E:\data\fruits.yaml') as file: # The FullLoader parameter handles the conversion from YAML # scalar values to Python the dictionary format # fruits_list = yaml.load(file, Loader=yaml.FullLoader) conn = connection.Connection(**data) banner("Connection") print (conn) servers = conn.list_servers() pprint (servers[0]['hostname']) #projects = conn.identity.projects() # #pprint(projects) #for project in projects: # pprint(project) banner("List Networks")
msg = "Hello World!" variables = Variables() print("") Console.ok(msg) print("") Console.error(msg) print("") Console.msg(msg) print("") Console.error(msg, prefix = True, traceflag = True) variables['debug'] = True variables['trace'] = True variables['verbose'] = 10 print("Using Banner") banner("Hello World!") print("Using Heading") class example(object): def doit(self): HEADING() print("Hello World!") doit("Hello World!") print("Verbose") d = {"sample" : "value"} VERBOSE(d)
def test_results(self): HEADING() # storage = self.p.service service = self.service banner(f"Benchmark results for {service} Storage") StopWatch.benchmark()
def run_script(self, name=None, hosts=None): banner(name) results = self.run(script=self.script[name], hosts=hosts, verbose=True)
def do_multipass(self, args, arguments): """ :: Usage: multipass list [--output=OUTPUT] [--dryrun] multipass images [--output=OUTPUT] [--dryrun] multipass start NAMES [--output=OUTPUT] [--dryrun] multipass stop NAMES [--output=OUTPUT] [--dryrun] multipass reboot NAMES [--output=OUTPUT] [--dryrun] multipass delete NAMES [--output=OUTPUT][--dryrun] multipass destroy NAMES [--output=OUTPUT][--dryrun] multipass shell NAMES [--dryrun] multipass run COMMAND NAMES [--output=OUTPUT] [--dryrun] multipass info NAMES [--output=OUTPUT] [--dryrun] multipass suspend NAMES [--output=OUTPUT] [--dryrun] multipass resume NAMES [--output=OUTPUT] [--dryrun] multipass destroy NAMES [--dryrun] multipass create NAMES [--image=IMAGE] [--size=SIZE] [--mem=MEMORY] [--cpus=CPUS] [--cloud-init=CLOUDINIT] [--dryrun] multipass reboot NAMES [--dryrun] multipass mount SOURCE DESTINATION [--dryrun] multipass umount SOURCE [--dryrun] multipass transfer SOURCE DESTINATION [--dryrun] multipass set key=VALUE [--dryrun] multipass get [key] [--dryrun] Interface to multipass Options: --output=OUTPUT the output format [default: table]. Other values are yaml, csv and json. --image=IMAGE the image name to be used to create a VM. Arguments: NAMES the names of the virtual machine Description: The NAMES can be a parameterized hostname such as red[0-1,5] = red0,red1,red5 Commands: First you can see the supported multipass images with cms multipass images Create and launch a new vm using cms multipass create NAMES Optionally you can provide image name, size, memory, # of cpus to create an instance. Start one or multiple multipass vms with cms multipass start NAMES Stop one or multiple vms with cms multipass stop NAMES Gets all multipass internal key values with cms multipass get Gets a specific internal key. cms multipass get KEY Known keys client.gui.autostart client.primary-name local.driver are there more? Reboot (stop and then start) vms with cms multipass reboot NAMES Delete one of multiple vms without purging with cms multipass delete NAMES Destory multipass vms (delete and purge) with cms multipass destroy NAMES Caution: Once destroyed everything in vm will be deleted and cannot be recovered. WHEN YOU IMPLEMENT A FUNCTION INCLUDE MINIMAL DOCUMENTATION HERE """ name = arguments.NAME map_parameters(arguments, "dryrun", "refresh", "cloud", "image", "size", "mem", "cpus", "output") # so we can use arguments.cloudinit arguments["cloudinit"] = arguments["--cloud-init"] image = arguments.image variables = Variables() arguments.output = Parameter.find("output", arguments, variables, "table") names = Parameter.expand(arguments.NAMES) VERBOSE(arguments) if arguments.list: if arguments.dryrun: banner("dryrun list") else: provider = Provider() provider.list() return "" elif arguments.images: if arguments.dryrun: banner("dryrun images") else: provider = Provider() images = provider.images() print( provider.Print(images, kind='image', output=arguments.output)) return "" elif arguments.run: if arguments.dryrun: banner("dryrun run") for name in names: if arguments.dryrun: Console.ok(f"run {name} {arguments.COMMAND}") else: provider = Provider() provider.run(name, arguments.COMMAND) return "" elif arguments.create: result = "" if arguments.dryrun: banner("create") for name in names: if arguments.dryrun: Console.ok(f"dryrun create {name} {image}") else: provider = Provider() result = provider.create(name, image) VERBOSE(result) return result elif arguments.start: result = "" if arguments.dryrun: banner("start") for name in names: if arguments.dryrun: Console.ok(f"dryrun start {name}") else: provider = Provider() result = provider.start(name) VERBOSE(result) return result elif arguments.stop: result = "" if arguments.dryrun: banner("stop") for name in names: if arguments.dryrun: Console.ok(f"dryrun stop {name}") else: provider = Provider(name=name) result = provider.stop(name) VERBOSE(result) return result elif arguments.delete: result = "" if arguments.dryrun: banner("delete") for name in names: if arguments.dryrun: Console.ok(f"dryrun delete {name}") else: provider = Provider() # Default purge is false. Is this ok? result = provider.delete(name) VERBOSE(result) return result elif arguments.info: result = "" if arguments.dryrun: banner(f"info {name}") for name in names: if arguments.dryrun: Console.ok(f"dryrun info {name}") else: provider = Provider() # Default purge is false. Is this ok? result = provider.info(name) VERBOSE(result) return result elif arguments.suspend: result = "" if arguments.dryrun: banner("suspend") for name in names: if arguments.dryrun: Console.ok(f"dryrun suspend {name}") else: provider = Provider() result = provider.suspend(name) VERBOSE(result) return result elif arguments.resume: result = "" if arguments.dryrun: banner("resume") for name in names: if arguments.dryrun: Console.ok(f"dryrun resume {name}") else: provider = Provider() result = provider.resume(name) VERBOSE(result) return result elif arguments.destroy: result = "" if arguments.dryrun: banner("destroy") for name in names: if arguments.dryrun: Console.ok(f"dryrun destroy {name}") else: provider = Provider() result = provider.destroy(name) VERBOSE(result) return result elif arguments.reboot: result = "" if arguments.dryrun: banner("reboot") for name in names: if arguments.dryrun: Console.ok(f"dryrun reboot {name}") else: provider = Provider() result = provider.reboot(name) VERBOSE(result) return result elif arguments.shell: if len(names) > 1: Console.error("shell must only have one host") return "" name = names[0] if arguments.dryrun: banner("dryrun shell {name}") else: provider = Provider() provider.shell() return "" elif arguments.info: if arguments.dryrun: banner("dryrun info") else: provider = Provider() info = provider.info() print( provider.Print(info, kind='info', output=arguments.output)) return "" elif arguments.mount: if arguments.dryrun: banner( f"dryrun mount {arguments.SOURCE} {arguments.DESTINATION}") else: provider = Provider() provider.mount(arguments.SOURCE, arguments.DESTINATION) # list the mounts and display as table return "" else: Console.error("Not yet implemented") return ""
def images(self): banner("images list") os.system("multipass find")
def create(self, name=None, image=None, size=None, location=None, timeout=360, key=None, secgroup=None, ip=None, user=None, public=None, group=None, metadata=None, **kwargs): """ creates a named node :param name: the name of the node :param image: the image used :param size: the size of the image :param timeout: a timeout in seconds that is invoked in case the image does not boot. The default is set to 3 minutes. :param kwargs: additional arguments passed along at time of boot :return: the list with the modified dicts """ """ create one node """ if not ip and public: ip = self.find_available_public_ip() elif ip is not None: entry = self.list_public_ips(ip=ip, available=True) if len(entry) == 0: Console.error("ip not available") return None banner("Create Server") Console.msg(f" Name: {name}") Console.msg(f" IP: {ip}") Console.msg(f" Image: {image}") Console.msg(f" Size: {size}") Console.msg(f" Public: {public}") Console.msg(f" Key: {key}") Console.msg(f" location:{location}") Console.msg(f" timeout: {timeout}") Console.msg(f" secgroup:{secgroup}") Console.msg(f" group: {group}") # Validate if there is any VM with same tag name and state other than Terminated. # If there is any VM, throw error # ec2_reservations = self.info(name)['Reservations'] # # if ec2_reservations: # reservation_instances = None # for reservation in ec2_reservations: # reservation_instances = list( # filter(lambda instance: instance['State']['Name'] != 'terminated', reservation['Instances'])) # # if reservation_instances: # Console.error("Tag name already exists, Please use different tag name.") # return if secgroup is None: secgroup = 'default' if key is None: raise ValueError("Key must be set. Use cms set key=<key name>") # # BUG: the tags seem incomplete # if metadata is None: metadata = [] metadata = [{ 'Key': 'cm.image', 'Value': image }, { 'Key': 'cm.name', 'Value': name }, { 'Key': 'cm.flavor', 'Value': size }, { 'Key': 'cm.user', 'Value': self.user }, { 'Key': 'cm.kind', 'Value': "vm" }, { 'Key': 'cm.status', 'Value': "BOOTING" }, { 'Key': 'Name', 'Value': name }] # VERBOSE(metadata) new_ec2_instance = self.ec2_resource.create_instances( ImageId=image, InstanceType=size, MaxCount=1, MinCount=1, SecurityGroups=[secgroup], KeyName=key, TagSpecifications=[{ 'ResourceType': 'instance', 'Tags': metadata }]) # VERBOSE(new_ec2_instance) new_ec2_instance = new_ec2_instance[0] waiter = self.ec2_client.get_waiter('instance_exists') waiter.wait(Filters=[{ 'Name': 'instance-id', 'Values': [new_ec2_instance.instance_id] }], WaiterConfig={ 'Delay': 20, 'MaxAttempts': timeout / 20 }) print() Console.ok("Instance created...") print() # if IP provided, Attach it to new instance if ip: self.attach_public_ip(name, ip) # x = self.ec2_client.describe_instances(InstanceIds=[new_ec2_instance.instance_id]) # VERBOSE(x) data = self.info(name=name) # VERBOSE(data) data['name'] = name data['kind'] = 'aws' data['status'] = new_ec2_instance.state['Name'], data['created'] = new_ec2_instance.launch_time.strftime( "%m/%d/%Y, %H:%M:%S") if new_ec2_instance.launch_time else '', data['updated'] = new_ec2_instance.launch_time.strftime( "%m/%d/%Y, %H:%M:%S") if new_ec2_instance.launch_time else '', data['name'] = new_ec2_instance.tags[0][ 'Value'] if new_ec2_instance.tags else '', data['instance_id'] = new_ec2_instance.id, data['image'] = new_ec2_instance.image_id, data['key_name'] = key, Console.msg("Waiting for the Public IP address assignment ...") while True: try: public_ip = \ self.ec2_client.describe_instances( InstanceIds=[new_ec2_instance.id])['Reservations'][0][ 'Instances'] \ [0]['PublicIpAddress'], break except KeyError: time.sleep(0.5) data['public_ips'] = public_ip[0] data['private_ips'] = new_ec2_instance.private_ip_address Console.msg(f" Public IP: {data['public_ips']}") Console.msg(f" Private IP: {data['private_ips']}") output = self.update_dict(data, kind="vm")[0] return output
def list(self): # list instances banner("list the launched instance") os.system("multipass list") print('\n')
def shell(self): banner("open shell on running instance") os.system(f"multipass shell") print('\n')
def delete(self): banner(f"delete {self.name}") # delete the instance os.system(f"multipass delete {self.name}")
def purge(self): banner(f"purge") os.system(f"multipass purge") print('\n')
def stop(self): banner(f"stop {self.name}") os.system(f"multipass stop {self.name}") print('\n')
def launch(self): banner(f"launch {self.name}") os.system(f"multipass launch -n {self.name}") print('\n')
def setup(self): banner("setup", c="-") self.user = Config()["cloudmesh.profile.user"] self.p = Provider(name="aws")
def do_config(self, args, arguments): """ :: Usage: config -h | --help config cat [less] config check config secinit config security add (--secret=REGEXP | --exception=REGEXP ) config security rmv (--secret=REGEXP | --exception=REGEXP ) config security list config encrypt config decrypt [--nopass] config edit [ATTRIBUTE] config set ATTRIBUTE=VALUE config get ATTRIBUTE [--output=OUTPUT] config value ATTRIBUTE config cloud verify NAME [KIND] config cloud edit [NAME] [KIND] config cloud list NAME [KIND] [--secrets] Arguments: SOURCE the file to encrypted or decrypted. an .enc is added to the filename or removed form it dependent of if you encrypt or decrypt ATTRIBUTE=VALUE sets the attribute with . notation in the configuration file. ATTRIBUTE reads the attribute from the container and sets it in the configuration file If the attribute is a password, * is written instead of the character included REGEXP python regular expression Options: --secret=REGEXP ensures all attributes within cloudmesh.yaml whose dot path matches REGEXP are not encrypted (even if listed in secrets) --exception=REGEXP ensures attributes within cloudmesh.yaml whose dot path matches REGEXP are encrypted --name=KEYNAME The name of a key --nopass Indicates if private key is password protected --output=OUTPUT The output format [default: yaml] --secrets Print the secrets. Use carefully. Description: config check checks if the ssh key ~/.ssh/id_rsa has a password. Verifies it through entering the passphrase Key generation Keys can be generated with cms key gen (ssh | pem) Key validity and password can be verified with cms key verify (ssh | pem) key verify (ssh | pem) [--filename=FILENAME] [--pub] ssh-add cms config encrypt Encrypts the config data at-rest. This means that the data is encrypted when not in use. This command is reliant upon the cloudmesh.security.secrets attribute and the cloudmesh.security.exceptions attribute within the cloudmesh.yaml file. Note, that the encrypted data is not encrypted upon request/query to the attribute. This means you must decrypt the config when needed in use and re-encrypt when not using the file, or delivering the file. 1. cloudmesh.security.secrets: This attribute will hold a list of python regular expressions that detail which attributes will be encrypted by the command. ex) .*: will encrypt all attributes ex) .*mdbpwd.*: will encrypt all paths with mdbpwd 2. cloudmesh.security.exceptions: This attribute will hold a list of python regular expressions that detail which attributes will not be encrypted by the command. ex) .*pubkey.*: ensures no pubkey path is encrypted security add --secret=REGEXP Adds valid REGEXP to the cloudmesh.security.secrets section security rmv --secret=REGEXP Removes REGEXP from the cloudmesh.security.secrets section security add --exception=REGEXP Adds valid REGEXP to cloudmesh.security.exceptions section security rmv --exception=REGEXP Removes REGEXP from cloudmesh.security.exceptions section security list Prints a list of all the attribute dot-paths that are referenced by cms config encryption and decryption commands cms config decrypt Decrypts the config data that was held in rest. This command decrypts and attributes that were encrypted using the sister `cms config encrypt` command. config set ATTRIBUTE=VALUE config set profile.name=Gregor In case the ATTRIBUTE is the name of a cloud defined under cloudmesh.cloud, the value will be written into the credentials attributes for that cloud this way you can safe a lot of typing. An example is cms config set aws.AWS_TEST=Gregor which would write the AWS_TEST attribute in the credentials of the cloud aws. This can naturally be used to set for example username and password. """ # d = Config() #~/.cloudmesh/cloudmesh.yaml # d = Config(encryted=True) # ~/.cloudmesh/cloudmesh.yaml.enc map_parameters(arguments, "exception", "keep", "nopass", "output", "secret", "secrets") source = arguments.SOURCE or path_expand("~/.cloudmesh/cloudmesh.yaml") destination = source + ".enc" if arguments.cloud and arguments.edit and arguments.NAME is None: path = path_expand("~/.cloudmesh/cloudmesh.yaml") print(path) Shell.edit(path) return "" cloud = arguments.NAME kind = arguments.KIND if kind is None: kind = "cloud" configuration = Config() if arguments.cloud and arguments.verify: service = configuration[f"cloudmesh.{kind}.{cloud}"] result = {"cloudmesh": {"cloud": {cloud: service}}} action = "verify" banner( f"{action} cloudmesh.{kind}.{cloud} in ~/.cloudmesh/cloudmesh.yaml" ) print(yaml.dump(result)) flat = flatten(service, sep=".") for attribute in flat: if "TBD" in str(flat[attribute]): Console.error( f"~/.cloudmesh.yaml: Attribute cloudmesh.{cloud}.{attribute} contains TBD" ) elif arguments.cloud and arguments.list: service = configuration[f"cloudmesh.{kind}.{cloud}"] result = {"cloudmesh": {"cloud": {cloud: service}}} action = "list" banner( f"{action} cloudmesh.{kind}.{cloud} in ~/.cloudmesh/cloudmesh.yaml" ) lines = yaml.dump(result).split("\n") secrets = not arguments.secrets result = Config.cat_lines(lines, mask_secrets=secrets) print(result) elif arguments.cloud and arguments.edit: # # there is a duplicated code in config.py for this # action = "edit" banner( f"{action} cloudmesh.{kind}.{cloud}.credentials in ~/.cloudmesh/cloudmesh.yaml" ) credentials = configuration[ f"cloudmesh.{kind}.{cloud}.credentials"] print(yaml.dump(credentials)) for attribute in credentials: if "TBD" in credentials[str(attribute)]: value = credentials[attribute] result = input(f"Please enter {attribute}[{value}]: ") credentials[attribute] = result # configuration[f"cloudmesh.{kind}.{cloud}.credentials"] = credentials print( yaml.dump( configuration[f"cloudmesh.{kind}.{cloud}.credentials"])) elif arguments["edit"] and arguments["ATTRIBUTE"]: attribute = arguments.ATTRIBUTE config = Config() config.edit(attribute) config.save() return "" elif arguments.cat: content = Config.cat() import shutil columns, rows = shutil.get_terminal_size(fallback=(80, 24)) lines = content.split("\n") counter = 1 for line in lines: if arguments.less: if counter % (rows - 2) == 0: x = input().split("\n")[0].strip() if x != '' and x in 'qQxX': return "" print(line) counter += 1 return "" elif arguments.check: Config.check() elif arguments.encrypt: config = Config() config.encrypt() elif arguments.decrypt: config = Config() config.decrypt(ask_pass=not arguments.nopass) elif arguments.set: config = Config() clouds = config["cloudmesh.cloud"].keys() line = arguments["ATTRIBUTE=VALUE"] attribute, value = line.split("=", 1) cloud, field = attribute.split(".", 1) if cloud in clouds: attribute = f"cloudmesh.cloud.{cloud}.credentials.{field}" elif not attribute.startswith("cloudmesh."): attribute = f"cloudmesh.{attribute}" config[attribute] = value config.save() elif arguments.value: config = Config() attribute = arguments.ATTRIBUTE if not attribute.startswith("cloudmesh."): attribute = f"cloudmesh.{attribute}" try: value = config[attribute] if type(value) == dict: raise Console.error("the variable is a dict") else: print(f"{value}") except Exception as e: print(e) return "" elif arguments.secinit: config = Config() secpath = path_expand(config['cloudmesh.security.secpath']) if not os.path.isdir(secpath): Shell.mkdir(secpath) # Use Shell that makes all dirs as needed elif arguments.security and arguments.list: config = Config() secrets = config.get_list_secrets() for s in secrets: Console.msg(s) elif arguments.security: # Get the regular expression from command line regexp = None if arguments.secret: regexp = arguments.secret elif arguments.exception: regexp = arguments.exception # Verify argument is valid python regular expression try: r = re.compile(regexp) except re.error: Console.error(f"Invalid Python RegExp:{regexp}") sys.exit() config = Config() path = None section = None # Assign information based on arguments if arguments.secret: path = 'cloudmesh.security.secrets' section = "secrets" elif arguments.exception: path = 'cloudmesh.security.exceptions' section = "exceptions" # Get current list of regular expressions from related section exps = config[path] # Add argument to expressions in related section if arguments.add: if regexp not in exps: config[path].append(regexp) config.save() Console.ok(f"Added {regexp} to {section}") else: Console.warning(f"{regexp} already in {section}") # Remove argument from expressions in related section elif arguments.rmv: if regexp in exps: config[path].remove(regexp) config.save() Console.ok(f"Removed {regexp} from {section}") else: Console.warning(f"{regexp} not in {section}") elif arguments.get: print() config = Config() clouds = config["cloudmesh.cloud"].keys() attribute = arguments.ATTRIBUTE try: cloud, field = attribute.split(".", 1) field = f".{field}" except: cloud = attribute field = "" if cloud in clouds: attribute = f"cloudmesh.cloud.{cloud}{field}" elif not attribute.startswith("cloudmesh."): attribute = f"cloudmesh.{attribute}" try: value = config[attribute] if type(value) == dict: print(Printer.write(value, output=arguments.output)) else: print(f"{attribute}={value}") except Exception as e: print(e) return "" return ""
def find(self): banner(f"find available multipass images") os.system(f"multipass find") print('\n')
def list(self): # list instances banner("list") os.system("multipass ls")
def create(self, name=None, image=None, size=None, location=None, timeout=360, key=None, secgroup=None, ip=None, user=None, public=True, group=None, metadata=None, cloud=None, **kwargs): """ creates a named node :param group: the list of groups the vm belongs to :param name: the name of the node :param image: the image used :param size: the size of the image :param timeout: a timeout in seconds that is invoked in case the image does not boot. The default is set to 3 minutes. :param kwargs: additional arguments HEADING(c=".")ed along at time of boot :return: """ image_use = None flavor_use = None # keyname = Config()["cloudmesh"]["profile"]["user"] # ex_keyname has to be the registered keypair name in cloud """ https://docs.openstack.org/openstacksdk/latest/user/connection.html#openstack.connection.Connection.create_server """ if 'flavor' in kwargs and size is None: size = kwargs['flavor'] # Guess user name if user is None: user = Image.guess_username(image) # image_name = image.lower() # if image_name.startswith("cc-"): # user = "******" # if "centos" in image_name: # user = "******" # elif "ubuntu" in image_name: # user = "******" # get IP if not ip and public: ip = self.find_available_public_ip() # pprint(entry) elif ip is not None: entry = self.list_public_ips(ip=ip, available=True) if len(entry) == 0: print("ip not available") raise ValueError(f"The ip can not be assigned {ip}") if type(group) == str: groups = Parameter.expand(group) banner("Create Server") print(" Name: ", name) print(" User: "******" IP: ", ip) print(" Image: ", image) print(" Size: ", size) print(" Public: ", public) print(" Key: ", key) print(" location:", location) print(" timeout: ", timeout) print(" secgroup:", secgroup) print(" group: ", group) print(" groups: ", groups) print() try: server = self.cloudman.create_server(name, flavor=size, image=image, key_name=key, security_groups=[secgroup], timeout=timeout # tags=groups, # wait=True ) server['user'] = user r = self.cloudman.wait_for_server(server) s = self.cloudman.add_ips_to_server(server, ips=ip) variables = Variables() variables['vm'] = name if metadata is None: metadata = {} metadata['image'] = image metadata['flavor'] = size self.cloudman.set_server_metadata(server, metadata) self.add_secgroup(name=secgroup) # server = self.cloudman.compute.wait_for_server(server) # print("ssh -i {key} root@{ip}".format( # key=PRIVATE_KEYPAIR_FILE, # ip=server.access_ipv4)) except Exception as e: Console.error("Problem starting vm", traceflag=True) print(e) raise RuntimeError return self.update_dict(server, kind="vm")[0]
# sp20-516-255 E.Cloudmesh.Common.1 # Program that demonstrates the use of banner, HEADING, and VERBOSE. from cloudmesh.common.util import banner from cloudmesh.common.util import HEADING from cloudmesh.common.debug import VERBOSE bannerText = "Banner Test" ## banner banner(bannerText, "*", color="BLUE") ## HEADING headingText = "Heading Test" HEADING(headingText) #VERBOSE verboseText = {"Banner": bannerText, "Heading": headingText} VERBOSE(verboseText)
def __init__(self, service=None, config="~/.cloudmesh/cloudmesh.yaml", **kwargs): super().__init__(service=service, config=config) if kwargs.get("debug"): print("Inside init of local provider") print(self.kind) print(kwargs.get('sourceObj')) print(kwargs.get('target')) print(kwargs.get('targetObj')) print(self.credentials) # Processing --source/service and --target arguments separately. # This is a provider class for local storage hence --source/service will \ # always be "local" self.sourceCSP = self.service try: self.config = Config(config_path=config) self.yaml_content_source = self.config["cloudmesh.storage." f"{self.sourceCSP}"] self.source_kind = self.yaml_content_source["cm"]["kind"] self.source_credentials = self.yaml_content_source["credentials"] print("Accessing local storage location.") if kwargs.get('sourceObj'): self.local_location = Path(self.yaml_content_source['default'][ 'directory'], kwargs.get('sourceObj')) else: self.local_location = self.yaml_content_source['default'][ 'directory'] if kwargs.get("debug"): print(f"\nLocal location to access {self.local_location}") if kwargs.get('target'): self.targetCSP = kwargs.get('target') self.yaml_content_target = self.config["cloudmesh.storage." f"{self.targetCSP}"] self.target_kind = self.yaml_content_target["cm"]["kind"] self.target_credentials = self.yaml_content_target[ "credentials"] self.target_container = self.target_credentials["container"] except Exception as e: Console.error(f"Couldn't access cloudmesh.yaml. Error - {e}") return () if kwargs.get("debug"): VERBOSE(self.yaml_content_source) if kwargs.get('target'): VERBOSE(self.yaml_content_target) banner(f"Source CSP: {self.source_kind}") if kwargs.get('target'): banner(f"Target CSP: {self.target_kind}") # Creating connection with the target CSP. This done only if the # --target argument is provided. Only "copy" command is expected to # have --target argument. if kwargs.get('target'): if self.target_kind == "azureblob": print("Create Azure connection.") if 'TBD' == self.target_credentials["access_key_id"] \ or 'TBD' == self.target_credentials["secret_access_key"] \ or 'TBD' == self.target_credentials["region"]: Console.error("Critical details missing from .yaml file. " "TBD not allowed. Please check.") try: self.s3_client = boto3.client( 's3', aws_access_key_id=self.target_credentials[ "access_key_id"], aws_secret_access_key=self.target_credentials[ "secret_access_key"], region_name=self.target_credentials["region"] ) Console.ok( f"Successful connection to {self.target_kind} is " f"made.") except ClientError as e: Console.error(e, prefix=True, traceflag=True) elif self.kind == "gcpbucket": print("Create GCP connection.") raise NotImplementedError else: raise NotImplementedError
def do_register(self, args, arguments): """ :: Usage: register aws yaml register aws [FILENAME] [--keep] register azure [FILENAME] [--keep] register google [FILENAME] [--keep] register chameleon [FILENAME] [--keep] register new [-v] KIND SERVICE NAME [ATTRIBUTES...] register list KIND SERVICE NAME [ATTRIBUTES...] This command adds the registration information the th cloudmesh yaml file. The permissions of theFILENAME will also be changed. A y/n question will be asked if the files with the filename should be deleted after integration Arguments: FILENAME a filename in which the cloud credentials are stored Options: --keep keeps the file with the filename. Description: AWS AWS dependent on how you downloaded the credentials will either use the filename `credentials.csv` or `accessKey.csv` Our command is smart provides some convenience functionality. 1. If either file is found in `~/Downloads`, it is moved to `~/.cloudmesh` and the permissions are changed 2. If such a file already exists there it will ask if it should be overwritten in case the content is not the same 3. The content of the file will be read to determine if it is likely to be an AWS credential 4. The credential will be added to the cloudmesh yaml file Azure (TODO: THIS HAS TO BE IMPLEMENTED) To use this command you must have an Azure account and Azure CLI installed. Mote deatils to this can be found at: TBD: include link This command leverages the Azure CLI to extract specific account parameters and update the `cloudmesh.yaml` configuration file. 1. When this command is run, the `az login` Azure CLI command is executed, prompting you to login to your azure account via your default browser, which should then redirect you back to your CLI. 2. The azure cli command for `az account show` is then referenced to pull the account's subscription id and tenant id. 3. Run the command `az ad sp create-for-rbac --name http://cloudmesh` to reference the application id and secret key. NOTE: If there is already a secret key reference in the `cloudmesh.yaml` file, it will be rendered useless once the new secret key is obtained; requiring you to ensure the key has been successfully applied in the next step to the configuration file and cms init is called. 4. The credentials will be added to the `~/.cloudmesh.yaml` configuration file. Google Is not yet implemented Chameleon Cloud is not yet implemented Register from template: register list KIND SERVICE NAME [ATTRIBUTES...] This command lists the template for a provider and fills it out with the attributes. It can be used to check what information would be added with the new command. The template is included in the Provider itself. Leaving the attributes off, just prints the template Examples: cms register list compute openstack ABC cms register list compute openstack ABC uri=TBD tenant=TBD region=TBD In the last example the values for uri, tennant, and region will be cahnged to TBD register new KIND SERVICE NAME ATTRIBUTES... A new cloud can be added to the existing cloudmesh.yaml file with thsi command. AN example is cms register -v compute aws ABC \ region=one \ EC2_ACCESS_ID=TBD EC2_SECRET_KEY=TBD EC2_SECRET_KEY=TBD This command can also be used to overwrite an existing entry """ if arguments.aws and arguments.yaml: AWSReg = importlib.import_module("cloudmesh.register.AWSRegister") AWSregisterer = AWSReg.AWSRegister() AWSregisterer.register() elif arguments.azure: if arguments.yaml: Registry = importlib.import_module( "cloudmesh.register.AzRegister") AZregisterer = Registry.AzRegister() AZregisterer.register() return "" elif arguments.google: from cloudmesh.google.storage.Provider import Provider banner("Read the specification from json and write to yaml file") path = path_expand(arguments.FILE_JSON or "~/.cloudmesh/google.json") name = arguments.storage or "google" Provider.json_to_yaml(name, filename=path) Console.error("deleting {path} not implemented") return "" elif arguments.chameleon: Console.error("not yet implemented") return "" elif arguments.list: kind = arguments.KIND service = arguments.SERVICE attributes = arguments.ATTRIBUTES name = arguments.NAME replacements = {'name': name, 'service': service} for attribute in attributes: key, value = attribute.split("=") replacements[key] = value if kind == "compute" and service == "openstack": from cloudmesh.compute.openstack.Provider import Provider elif kind == "compute" and service == "azure": from cloudmesh.compute.azure.Provider import Provider elif kind == "compute" and service == "aws": from cloudmesh.compute.aws.Provider import Provider elif kind == "compute" and service == "oracle": from cloudmesh.oracle.compute.Provider import Provider elif kind == "storage" and service == "google": from cloudmesh.google.storage.Provider import Provider sample = Provider.sample try: if len(attributes) > 0: sample = sample.format(**replacements) except KeyError as e: Console.error(f"Value for {e} is not specified") print(dedent(sample)) elif arguments.new: kind = arguments.KIND service = arguments.SERVICE attributes = arguments.ATTRIBUTES name = arguments.NAME replacements = {'name': name, 'service': service} for attribute in attributes: key, value = attribute.split("=") replacements[key] = value if kind == "compute" and service == "openstack": from cloudmesh.compute.openstack.Provider import Provider elif kind == "compute" and service == "azure": from cloudmesh.compute.azure.Provider import Provider elif kind == "compute" and service == "aws": from cloudmesh.compute.aws.Provider import Provider elif kind == "compute" and service == "oracle": from cloudmesh.oracle.compute.Provider import Provider elif kind == "storage" and service == "google": from cloudmesh.google.storage.Provider import Provider else: Console.error("The provider {google} for {kind}" " does not have a sample configuration") return "" if kind in ["compute", "cloud"]: kind = "cloud" if kind in ["storage"]: kind = "storage" else: Console.error("the kind {kind} is not supported") return "" sample = Provider.sample try: sample = sample.format(**replacements) if arguments["-v"]: print(sample) Entry.add(entry=sample, base=f"cloudmesh.{kind}", path="~/.cloudmesh/cloudmesh.yaml") except KeyError as e: Console.error(f"Value for {e} is not specified") print(dedent(sample)) return ""
def setup(self): self.i = Inventory() banner("Info") self.i.info()
def update_slaves(self,hosts): banner("Updating $SPARK_HOME/conf/slaves file") command5 = f"echo 'pi@{hosts}' >> $SPARK_HOME/conf/slaves " print(command5) os.system(command5)
def showBanner(self, msg): banner(msg)
# fa20-516-238 E.Cloudmesh.Common.3 from cloudmesh.common.util import banner from cloudmesh.common.FlatDict import FlatDict class CloudmeshCommon: def demo_flatdict(self): test_dict = { "country": "UNITED STATES OF AMERICA", "attributes": { "Language": "English", "Capital": "Washington D.C.", "Currency": "USD" } } print(test_dict) flat_list = FlatDict(test_dict, sep=".") banner("Flattened list for test_dict") print(flat_list) if __name__ == '__main__': demo = CloudmeshCommon() banner("FlatDict Demo") demo.demo_flatdict()
# sp20-516-239 E.Cloudmesh.Common.1 from cloudmesh.common.util import banner banner("This is my banner") from cloudmesh.common.util import HEADING HEADING("Spring 2020 Cloud Computing Class") print("Demonstration of heading") from cloudmesh.common.debug import VERBOSE m = {"first_name": "Sara"} VERBOSE(m)
def test_banner(self): banner("START", color="RED")
# fa20-516-238 E.Cloudmesh.Common.2 from cloudmesh.common.util import banner from cloudmesh.common.dotdict import dotdict class CloudmeshCommon: def demo_dotdict(self): student = {"name": "Ishan Mishra", "id": "238", "subject": "ECC"} student = dotdict(student) print("Student Name:", student.name, "\n") print("Student ID:", student.id, "\n") print("Subject Enrolled:", student.subject, "\n") if __name__ == '__main__': demo = CloudmeshCommon() banner("\t\t Student's Details", c="x", prefix="@") demo.demo_dotdict()
from cloudmesh.common.util import banner, HEADING from cloudmesh.common.debug import VERBOSE banner("This is my first print command", debug=True, label="Label for Banner", color="GREEN") banner("Trying out Heading") HEADING("This is interesting!", c='#', color='HEADER') banner("This is Verbose") d = { "January": "1", "February": "2", "March": "3", "April": "4", "May": "5", "June": "6", "July": "7", "August": "8", "September": "9", "October": "10", "November": "11", "December": '12' } VERBOSE(d, label="Months", color='RED', verbose=-10)
def add(self, n1, n2): HEADING() result = n1 + n2 return result def divide(self, n1, n2): HEADING() result = 0 try: result = n1 / n2 except ZeroDivisionError as err: self.operErrors.append({ "operation": "divide", "Error": f"Error when dividing {n1} by {n2} : {err}" }) return result if __name__ == '__main__': mathUtil = MathUtil() banner("Start:: Testing Math Util") sum = mathUtil.add(3, 5) print(sum) div = mathUtil.divide(50, 0) banner("End:: Testing Math Util") VERBOSE(mathUtil.operErrors[0])