def do_pi(self, args, arguments): """ :: Usage: pi led reset [NAMES] pi led (red|green) VALUE pi led (red|green) VALUE NAMES [--user=USER] pi led list NAMES [--user=USER] pi led blink (red|green) NAMES [--user=USER] [--rate=SECONDS] pi led sequence (red|green) NAMES [--user=USER] [--rate=SECONDS] pi temp NAMES [--rate=RATE] [--user=USER] [--output=FORMAT] pi free NAMES [--rate=RATE] [--user=USER] [--output=FORMAT] pi load NAMES [--rate=RATE] [--user=USER] [--output=FORMAT] pi spark setup --master=MASTER --workers=WORKERS pi spark start --master=MASTER --workers=WORKERS pi spark stop --master=MASTER --workers=WORKERS pi spark test --master=MASTER --workers=WORKERS pi spark check [--master=MASTER] [--workers=WORKERS] pi k3 install [--master=MASTER] [--workers=WORKERS] [--step=COMMAND] pi k3 join --master=MASTER --workers=WORKERS pi k3 uninstall [--master=MASTER] [--workers=WORKERS] pi k3 delete [--master=MASTER] [--workers=WORKERS] pi k3 test [--master=MASTER] [--workers=WORKERS] pi k3 view pi script list SERVICE [--details] pi script list SERVICE NAMES pi script list pi wifi SSID [PASSWORD] [--dryrun] This command does some useful things. Arguments: FILE a file name Options: -f specify the file Description: This command switches on and off the LEDs of the specified PIs. If the hostname is ommitted. IT is assumed that the code is executed on a PI and its LED are set. To list the PIs LED status you can use the list command Examples: cms pi led list "red,red[01-03]" lists the LED status of the given hosts cms pi led red off "red,red[01-03]" switches off the led of the given PIs cms pi led red on "red,red[01-03]" switches on the led of the given PIs cms pi led red blink "red,red[01-03]" switches on and off the led of the given PIs cms pi led red sequence "red,red[01-03]" goes in sequential order and switches on and off the led of the given PIs """ map_parameters(arguments, 'details', 'master', 'workers', 'output', 'user', 'rate') arguments.output = arguments.output or 'table' if arguments.free: free = Free() free.execute(arguments) elif arguments.temp: temp = Temperature() temp.execute(arguments) elif arguments.load: load = Load() load.execute(arguments) elif arguments.led: led = LED() led.execute(arguments) elif arguments.spark: from cloudmesh.cluster.spark.spark import Spark spark = Spark() spark.execute(arguments) elif arguments.k3: from cloudmesh.cluster.k3.k3 import K3 k3 = K3() k3.execute(arguments) elif arguments.script: script = Script() script.execute(arguments) elif arguments.wifi: wifi = Wifi() if not wifi.is_root(): Console.error("You are not running in sudo") return "" if arguments.PASSWORD is None: arguments.PASSWORD = getpass( f"Wifi Password for {arguments.SSID}: ") wifi.set(arguments.SSID, arguments.PASSWORD, dryrun=arguments["--dryrun"]) return ""
def install(self, sudo=True): """ check where the MongoDB is installed in mongo location. if MongoDB is not installed, python help install it """ if self.dryrun: print(self.mongo_path) # pprint(self.data) mode = self.data['MODE'] Console.msg(f"Installing mongo in mode: {mode}") if mode == 'docker': Console.ok( "Installing mongoDB in a docker container cloudmesh-mongo") from cloudmesh.mongo.MongoDocker import MongoDocker mongo = MongoDocker() mongo.kill() mongo.install(clean=True, pull=True) Console.ok("Shutting mongoDB down") Console.msg("") Console.ok("Start the mongodb service with") Console.msg("") Console.msg(" cms admin mongo create") Console.msg(" cms admin mongo start") Console.msg("") return "" if not self.data["MONGO_AUTOINSTALL"]: Console.error("Mongo auto install is off") print("You can set it with") print() Console.ok( " cms config set cloudmesh.data.mongo.MONGO_AUTOINSTALL=True" ) print() if self.machine == 'darwin': print("To install it with brew you need to set also") print() Console.ok( " cms config set cloudmesh.data.mongo.MONGO_BREWINSTALL=True" ) print() return "" # # the path test may be wrong as we need to test for mongo and mongod # # print ('OOO', os.path.isdir(path), self.data["MONGO_AUTOINSTALL"] ) if self.force or (not os.path.isdir(self.mongo_home) and self.data["MONGO_AUTOINSTALL"]): print(f"MongoDB is not installed in {self.mongo_home}") # # ask if you like to install and give info where it is being installed # # use cloudmesh yes no question see cloudmesh 3 # # print(f"Auto-install the MongoDB into {mongo_path}") self.local = self.data["LOCAL"] if self.machine == 'linux': self.linux() elif self.machine == 'darwin': self.darwin() elif self.machine == 'win32': # Replaced windows with win32 self.windows() else: print("platform not found", platform) elif os.path.isdir(self.mongo_home): Console.error(f"Folder {self.mongo_home} already exists")
def do_open(self, args, arguments): """ :: Usage: open chameleon baremetal tacc open chameleon baremetal uc open chameleon vm open chameleon openstack open FILENAME open doc local open doc open git REPO open account aws [NAME] Arguments: FILENAME the file to open in the cwd if . is specified. If file in in cwd you must specify it with ./FILENAME if the FILENAME is doc than teh documentation from the Web is opened. Description: Opens the given URL in a browser window. open chameleon baremetal tacc starts horizon for baremetal for chameleon cloud at TACC open chameleon baremetal uc starts horizon for baremetal for chameleon cloud at UC open chameleon vm starts horizon for virtual machines open account aws [NAME] opens the aws users web page, if the NAME is users or is omitted, it goes to the page that allows you to create a user """ # pprint(arguments) filename = arguments.FILENAME if arguments.git and arguments.REPO: try: from cloudmesh_installer.install.installer import Git filename = Git.url(arguments.REPO) except ModuleNotFoundError: Console.error("To run this command you need to install " "cloudmesh-installer.\n" " Install it with\n\n" " pip install cloudmesh-installer -U\n") return "" elif arguments.aws and arguments.account: name = arguments.NAME or "users" if name == "users": filename = f"https://console.aws.amazon.com/iam/home#/users" else: filename = "https://console.aws.amazon.com/iam/home#/users" \ f"/{name}?section=security_credentials" elif arguments.baremetal and arguments.tacc: filename = "https://chi.tacc.chameleoncloud.org" elif arguments.baremetal and arguments.uc: filename = "https://chi.uc.chameleoncloud.org" elif arguments.chameleon and (arguments.vm or arguments.openstack): filename = "https://openstack.tacc.chameleoncloud.org" elif arguments.doc and arguments.local: filename = "./docs/index.html" elif filename == "doc": filename = "https://cloudmesh.github.io/cloudmesh-manual/" if not (filename.startswith("file:") or filename.startswith("http")): if not filename.startswith(".") and not filename.startswith("/"): filename = "./" + filename filename = path_expand(filename) if os.path.isfile(Path(filename)): filename = "file://" + filename else: Console.error("can not find the file {0}".format(filename)) return "" Console.ok("open {0}".format(filename)) try: webbrowser.open("%s" % filename) except Exception as e: Console.error( "can not open browser with file {0}".format(filename)) return ""
def do_config(self, args, arguments): """ :: Usage: config -h | --help config cat [less] config check config encrypt [SOURCE] [--keep] config decrypt [SOURCE] config edit [ATTRIBUTE] config set ATTRIBUTE=VALUE config get ATTRIBUTE [--output=OUTPUT] config value ATTRIBUTE config ssh keygen config ssh verify config ssh check config ssh pem config cloud verify NAME [KIND] config cloud edit [NAME] [KIND] config cloud list NAME [KIND] [--secrets] Arguments: SOURCE the file to encrypted or decrypted. an .enc is added to the filename or removed form it dependent of if you encrypt or decrypt ATTRIBUTE=VALUE sets the attribute with . notation in the configuration file. ATTRIBUTE reads the attribute from the container and sets it in the configuration file If the attribute is a password, * is written instead of the character included Options: --name=KEYNAME The name of a key --output=OUTPUT The output format [default: yaml] --secrets Print the secrets. Use carefully. Description: config check checks if the ssh key ~/.ssh/id_rsa has a password. Verifies it through entering the passphrase Key generation Keys must be generated with ssh-keygen -t rsa -m pem openssl rsa -in ~/.ssh/id_rsa -out ~/.ssh/id_rsa.pem or cms config ssh keygen Key validity can be checked with cms config check The key password can be verified with cms config verify ssh-add cms config encrypt ~/.cloudmesh/cloudmesh.yaml cms config decrypt ~/.cloudmesh/cloudmesh.yaml config set ATTRIBUTE=VALUE config set profile.name=Gregor In case the ATTRIBUTE is the name of a cloud defined under cloudmesh.cloud, the value will be written into the credentials attributes for that cloud this way you can safe a lot of typing. An example is cms config set aws.AWS_TEST=Gregor which would write the AWS_TEST attribute in the credentials of the cloud aws. This can naturally be used to set for example username and password. """ # d = Config() #~/.cloudmesh/cloudmesh.yaml # d = Config(encryted=True) # ~/.cloudmesh/cloudmesh.yaml.enc map_parameters(arguments, "keep", "secrets", "output") source = arguments.SOURCE or path_expand("~/.cloudmesh/cloudmesh.yaml") destination = source + ".enc" if arguments.cloud and arguments.edit and arguments.NAME is None: path = path_expand("~/.cloudmesh/cloudmesh.yaml") print(path) Shell.edit(path) return "" cloud = arguments.NAME kind = arguments.KIND if kind is None: kind = "cloud" configuration = Config() if arguments.cloud and arguments.verify: service = configuration[f"cloudmesh.{kind}.{cloud}"] result = {"cloudmesh": {"cloud": {cloud: service}}} action = "verify" banner( f"{action} cloudmesh.{kind}.{cloud} in ~/.cloudmesh/cloudmesh.yaml" ) print(yaml.dump(result)) flat = flatten(service, sep=".") for attribute in flat: if "TBD" in str(flat[attribute]): Console.error( f"~/.cloudmesh.yaml: Attribute cloudmesh.{cloud}.{attribute} contains TBD" ) elif arguments.cloud and arguments.list: service = configuration[f"cloudmesh.{kind}.{cloud}"] result = {"cloudmesh": {"cloud": {cloud: service}}} action = "list" banner( f"{action} cloudmesh.{kind}.{cloud} in ~/.cloudmesh/cloudmesh.yaml" ) lines = yaml.dump(result).split("\n") secrets = not arguments.secrets result = Config.cat_lines(lines, mask_secrets=secrets) print(result) elif arguments.cloud and arguments.edit: # # there is a duplicated code in config.py for this # action = "edit" banner( f"{action} cloudmesh.{kind}.{cloud}.credentials in ~/.cloudmesh/cloudmesh.yaml" ) credentials = configuration[ f"cloudmesh.{kind}.{cloud}.credentials"] print(yaml.dump(credentials)) for attribute in credentials: if "TBD" in credentials[str(attribute)]: value = credentials[attribute] result = input(f"Please enter {attribute}[{value}]: ") credentials[attribute] = result # configuration[f"cloudmesh.{kind}.{cloud}.credentials"] = credentials print( yaml.dump( configuration[f"cloudmesh.{kind}.{cloud}.credentials"])) elif arguments["edit"] and arguments["ATTRIBUTE"]: attribute = arguments.ATTRIBUTE config = Config() config.edit(attribute) config.save() return "" elif arguments.cat: content = Config.cat() import shutil columns, rows = shutil.get_terminal_size(fallback=(80, 24)) lines = content.split("\n") counter = 1 for line in lines: if arguments.less: if counter % (rows - 2) == 0: x = input().split("\n")[0].strip() if x != '' and x in 'qQxX': return "" print(line) counter += 1 return "" elif arguments.check and not arguments.ssh: Config.check() elif arguments.encrypt: e = EncryptFile(source, destination) e.encrypt() Console.ok(f"{source} --> {destination}") if not arguments.keep: os.remove(source) Console.ok("file encrypted") return "" elif arguments.decrypt: if ".enc" not in source: source = source + ".enc" else: destination = source.replace(".enc", "") if not os.path.exists(source): Console.error(f"encrypted file {source} does not exist") sys.exit(1) if os.path.exists(destination): Console.error( f"decrypted file {destination} does already exist") sys.exit(1) e = EncryptFile(source, destination) e.decrypt(source) Console.ok(f"{source} --> {source}") Console.ok("file decrypted") return "" elif arguments.ssh and arguments.verify: e = EncryptFile(source, destination) e.pem_verify() elif arguments.ssh and arguments.check: e = EncryptFile(source, destination) key = path_expand("~/.ssh/id_rsa") r = e.check_key(key) if r: Console.ok(f"Key {key} is valid") # does not work as it does not change it to pem format # e.check_passphrase() elif arguments.ssh and arguments.pem: e = EncryptFile(source, destination) r = e.pem_create() elif arguments.set: config = Config() clouds = config["cloudmesh.cloud"].keys() line = arguments["ATTRIBUTE=VALUE"] attribute, value = line.split("=", 1) cloud, field = attribute.split(".", 1) if cloud in clouds: attribute = f"cloudmesh.cloud.{cloud}.credentials.{field}" elif not attribute.startswith("cloudmesh."): attribute = f"cloudmesh.{attribute}" config[attribute] = value config.save() elif arguments.value: config = Config() attribute = arguments.ATTRIBUTE if not attribute.startswith("cloudmesh."): attribute = f"cloudmesh.{attribute}" try: value = config[attribute] if type(value) == dict: raise Console.error("the variable is a dict") else: print(f"{value}") except Exception as e: print(e) return "" elif arguments.get: print() config = Config() clouds = config["cloudmesh.cloud"].keys() attribute = arguments.ATTRIBUTE try: cloud, field = attribute.split(".", 1) field = f".{field}" except: cloud = attribute field = "" if cloud in clouds: attribute = f"cloudmesh.cloud.{cloud}{field}" elif not attribute.startswith("cloudmesh."): attribute = f"cloudmesh.{attribute}" try: value = config[attribute] if type(value) == dict: print(Printer.write(value, output=arguments.output)) else: print(f"{attribute}={value}") except Exception as e: print(e) return "" elif arguments.ssh and arguments.keygen: e = EncryptFile(source, destination) e.ssh_keygen() return ""
def _create(self, **kwargs): """ Create a volume. :param name (string): name of volume :param region (string): availability-zone :param encrypted (boolean): True|False :param size (integer): size of volume. Minimum size for st1 and sc1 is 500 GB. :param volume_type (string): type of volume. This can be gp2 for General Purpose SSD, io1 for Provisioned IOPS SSD (not implemented), st1 for Throughput Optimized HDD, sc1 for Cold HDD, or standard for Magnetic volumes. :param snapshot (string): snapshot id :return: dict """ if kwargs['volume_type'] == 'io1': raise NotImplementedError if kwargs['volume_type'] in ['sc1', 'st1']: if int(kwargs['size']) < 500: Console.error("minimum volume size for sc1 is 500 GB") if kwargs['snapshot'] != "None": r = self.client.create_volume( AvailabilityZone=kwargs['region'], Encrypted=kwargs['encrypted'], Size=int(kwargs['size']), SnapshotId=kwargs['snapshot'], VolumeType=kwargs['volume_type'], TagSpecifications=[ { 'ResourceType': 'volume', 'Tags': [ { 'Key': "Name", 'Value': kwargs['NAME'] }, ] }, ], ) else: r = self.client.create_volume( AvailabilityZone=kwargs['region'], Encrypted=kwargs['encrypted'], Size=int(kwargs['size']), VolumeType=kwargs['volume_type'], TagSpecifications=[ { 'ResourceType': 'volume', 'Tags': [ { 'Key': "Name", 'Value': kwargs['NAME'] }, ] }, ], ) r = [r] result = {'Volumes': r} result['Volumes'][0]['AttachedToVm'] = [] return result
def do_storage(self, args, arguments): """ :: Usage: storage [--storage=SERVICE] put FILENAME SOURCEDIR storage [--storage=SERVICE] get FILENAME DESTDIR storage [--storage=SERVICE] delete file FILENAME storage [--storage=SERVICE] list file DIRNAME storage [--storage=SERVICE] info FILENAME storage [--storage=SERVICE] create dir DIRNAME storage [--storage=SERVICE] list dir storage [--storage=SERVICE] delete dir DIRNAME This command does some useful things. Arguments: FILENAME a BLOB name SOURCEDIR local path for the FILENAME to be uploaded DESTDIR local path for the FILENAME to be downloaded Options: --storage=SERVICE specify the cloud service name like aws or azure or box or google Description: commands used to upload, download, list files on different cloud storage services. storage put [options..] Uploads the file specified in the filename to specified cloud from the SOURCEDIR. storage get [options..] Downloads the file specified in the filename from the specified cloud to the DESTDIR. storage delete file [options..] Deletes the file specified in the filename from the specified cloud. storage list file [options..] lists all the files from the container name specified on the specified cloud. storage info [options..] returns the properties of the filename specified on the specified cloud. storage create dir [options..] creates a folder with the directory name specified on the specified cloud. storage list dir [options..] lists all the folders on the specified cloud. storage delete dir [options..] deletes all the files in the directory specified on the specified cloud. Example: set storage=azureblob storage put FILENAME SOURCEDIR is the same as storage --storage=azureblob put FILENAME SOURCEDIR """ # arguments.CONTAINER = arguments["--container"] arguments.SERVICE = arguments["--storage"] pprint(arguments) m = Manager() service = None try: service = arguments["--storage"][0] except Exception as e: try: v = Variables() service = v['storage'] except Exception as e: service = None if service is None: Console.error("storage service not defined") if arguments['get']: if arguments.SERVICE is None: variables = Variables() arguments.SERVICE = variables['storage'] m.get(arguments.SERVICE, arguments.FILENAME, arguments.DESTDIR) elif arguments['put']: if arguments.SERVICE is None: variables = Variables() arguments.SERVICE = variables['storage'] m.put(arguments.SERVICE, arguments.FILENAME, arguments.SOURCEDIR) elif arguments['delete'] and arguments['file']: if arguments.SERVICE is None: variables = Variables() arguments.SERVICE = variables['storage'] m.delete(arguments.SERVICE, arguments.FILENAME) elif arguments['list'] and arguments['file']: if arguments.SERVICE is None: variables = Variables() arguments.SERVICE = variables['storage'] m.listfiles(arguments.SERVICE, arguments.DIRNAME) elif arguments['info']: if arguments.SERVICE is None: variables = Variables() arguments.SERVICE = variables['storage'] m.info(arguments.SERVICE, arguments.FILENAME) elif arguments['create'] and arguments['dir']: if arguments.SERVICE is None: variables = Variables() arguments.SERVICE = variables['storage'] m.createdir(arguments.SERVICE, arguments.DIRNAME) elif arguments['list'] and arguments['dir']: if arguments.SERVICE is None: variables = Variables() arguments.SERVICE = variables['storage'] m.listdir(arguments.SERVICE) elif arguments['delete'] and arguments['dir']: if arguments.SERVICE is None: variables = Variables() arguments.SERVICE = variables['storage'] m.deletedir(arguments.SERVICE, arguments.DIRNAME)
def execute(cls, cmd, arguments="", shell=False, cwd=None, traceflag=True, witherror=True): """Run Shell command :param witherror: if set to False the error will not be printed :param traceflag: if set to true the trace is printed in case of an error :param cwd: the current working directory in whcih the command is supposed to be executed. :param shell: if set to true the subprocess is called as part of a shell :param cmd: command to run :param arguments: we do not know yet :return: """ # print "--------------" result = None terminal = cls.terminal_type() # print cls.command os_command = [cmd] if terminal in ['linux', 'windows']: os_command = [cmd] elif 'cygwin' in terminal: if not cls.command_exists(cmd): print("ERROR: the command could not be found", cmd) return else: os_command = [cls.command[cls.operating_system()][cmd]] if isinstance(arguments, list): os_command = os_command + arguments elif isinstance(arguments, tuple): os_command = os_command + list(arguments) elif isinstance(arguments, str): os_command = os_command + arguments.split() else: print("ERROR: Wrong parameter type", type(arguments)) if cwd is None: cwd = os.getcwd() try: if shell: result = subprocess.check_output(os_command, stderr=subprocess.STDOUT, shell=True, cwd=cwd) else: result = subprocess.check_output( os_command, # shell=True, stderr=subprocess.STDOUT, cwd=cwd) except: if witherror: Console.error("problem executing subprocess", traceflag=traceflag) if result is not None: result = result.strip().decode() return result
def do_storage(self, args, arguments): """ :: Usage: storage create dir DIRECTORY [--storage=SERVICE] [--parallel=N] storage get SOURCE DESTINATION [--recursive] [--storage=SERVICE] [--parallel=N] storage put SOURCE DESTINATION [--recursive] [--storage=SERVICE] [--parallel=N] storage list [SOURCE] [--recursive] [--parallel=N] [--output=OUTPUT] [--dryrun] storage delete SOURCE [--parallel=N] [--dryrun] storage search DIRECTORY FILENAME [--recursive] [--storage=SERVICE] [--parallel=N] [--output=OUTPUT] storage sync SOURCE DESTINATION [--name=NAME] [--async] [--storage=SERVICE] storage sync status [--name=NAME] [--storage=SERVICE] storage config list [--output=OUTPUT] storage [--parallel=N] copy SOURCE DESTINATION [--recursive] storage copy --source=SOURCE:SOURCE_FILE_DIR --target=TARGET:TARGET_FILE_DIR This command does some useful things. Arguments: SOURCE SOURCE can be a directory or file DESTINATION DESTINATION can be a directory or file DIRECTORY DIRECTORY refers to a folder on the cloud service SOURCE:SOURCE_FILE_DIR source provider name: file or directory name TARGET:SOURCE_FILE_DIR destination provider name Options: --storage=SERVICE specify the cloud service name like aws or azure or box or google Description: commands used to upload, download, list files on different cloud storage services. storage put [options..] Uploads the file specified in the filename to specified cloud from the SOURCEDIR. storage get [options..] Downloads the file specified in the filename from the specified cloud to the DESTDIR. storage delete [options..] Deletes the file specified in the filename from the specified cloud. storage list [options..] lists all the files from the container name specified on the specified cloud. storage create dir [options..] creates a folder with the directory name specified on the specified cloud. storage search [options..] searches for the source in all the folders on the specified cloud. sync SOURCE DESTINATION puts the content of source to the destination. If --recursive is specified this is done recursively from the source If --async is specified, this is done asynchronously If a name is specified, the process can also be monitored with the status command by name. If the name is not specified all date is monitored. sync status The status for the asynchronous sync can be seen with this command config list Lists the configures storage services in the yaml file storage copy SOURCE DESTINATION Copies files from source storage to destination storage. The syntax of SOURCE and DESTINATION is: SOURCE - awss3:source.txt DESTINATION - azure:target.txt Description of the copy command: Command enables to Copy files between different cloud service providers, list and delete them. This command accepts `aws` , `google` and `local` as the SOURCE and TARGET provider. cms storage copy --source=SERVICE:SOURCE --target=DEST:TARGET Command copies files or directories from Source provider to Target Provider. cms storage slist --source=SERVICE:SOURCE Command lists all the files present in SOURCE provider's in the given SOURCE_FILE_DIR location This command accepts `aws` or `google` as the SOURCE provider cms storage sdelete --source=SERVICE:SOURCE Command deletes the file or directory from the SOURCE provider's SOURCE_FILE_DIR location Examples: cms storage_service copy --source=local:test1.txt --target=aws:uploadtest1.txt cms storage_service list --source=google:test cms storage_service delete --source=aws:uploadtest1.txt Example: set storage=azureblob storage put SOURCE DESTINATION --recursive is the same as storage --storage=azureblob put SOURCE DESTINATION --recursive storage copy azure:source.txt oracle:target.txt """ # arguments.CONTAINER = arguments["--container"] VERBOSE(arguments) map_parameters(arguments, "dryrun", "recursive", "storage", "source", "target") source = arguments.source target = arguments.target variables = Variables() VERBOSE(arguments) arguments.storage = Parameter.expand(arguments.storage) if arguments["get"]: provider = Provider(arguments.storage[0]) result = provider.get(arguments.SOURCE, arguments.DESTINATION, arguments.recursive) elif arguments.put: provider = Provider(arguments.storage[0]) result = provider.put(arguments.SOURCE, arguments.DESTINATION, arguments.recursive) elif arguments.create and arguments.dir: provider = Provider(arguments.storage[0]) result = provider.create_dir(arguments.DIRECTORY) elif arguments.list: """ storage list SOURCE [--parallel=N] """ sources = arguments.SOURCE or variables["storage"] or 'local:.' sources = Parameter.expand(sources) deletes = [] for source in sources: storage, entry = Parameter.separate(source) storage = storage or "local" deletes.append((storage, entry)) _sources = ', '.join(sources) for delete in deletes: service, entry = delete if arguments.dryrun: print(f"Dryrun: list {service}:{entry}") else: provider = Provider(service=service) provider.list(name=entry) return "" elif arguments.delete: """ storage delete SOURCE [--parallel=N] """ sources = arguments.SOURCE or variables["storage"] or 'local:.' sources = Parameter.expand(sources) deletes = [] for source in sources: storage, entry = Parameter.separate(source) storage = storage or "local" deletes.append((storage, entry)) _sources = ', '.join(sources) answer = yn_choice(f"Would you like to delete {_sources}?", default="no") if answer: for delete in deletes: service, entry = delete if arguments.dryrun: print(f"Dryrun: delete {service}:{entry}") else: provider = Provider(service=service) provider.delete(name=entry) else: Console.error("Deletion canceled") return "" elif arguments.search: for storage in arguments.storage: provider = Provider(storage) provider.search(arguments.DIRECTORY, arguments.FILENAME, arguments.recursive) elif arguments.rsync: # TODO: implement raise NotImplementedError elif arguments.copy: VERBOSE(f"COPY: Executing Copy command from {arguments.SOURCE} to " f"{arguments.DESTINATION} providers") print(f"DEBUG storage.py: INITIALIZE with {arguments.storage[0]} " "provider.") provider = Provider(arguments.storage[0]) result = provider.copy(arguments.SOURCE, arguments.DESTINATION, arguments.recursive) elif arguments.copy: scloud, sbucket = source.split(":", 1) or None tcloud, tbucket = target.split(":", 1) or None # print(scloud + " " + tcloud + " " + sbucket + " " + tbucket) if scloud == "aws" or scloud == "google": provider = Provider(service=scloud) provider.copy(scloud, tcloud, sbucket, tbucket) elif (scloud == "local" and tcloud == "aws") or (scloud == "local" and tcloud == "google"): provider = Provider(service=tcloud) provider.copy(scloud, tcloud, sbucket, tbucket) else: print("Not Implemented") return ""
def do_openapi3(self, args, arguments): """ :: Usage: openapi3 generate FUNCTION [YAML] --baseurl=BASEURL --filename=FILENAME --yamldirectory=DIRECTORY [--verbose] openapi3 server start YAML [NAME] [--directory=DIRECTORY] [--port=PORT] [--server=SERVER] [--verbose] [--debug] [--debug] [--fg] [--os] openapi3 server stop NAME openapi3 server list [NAME] [--output=OUTPUT] openapi3 server ps [NAME] [--output=OUTPUT] openapi3 register add NAME ENDPOINT openapi3 register filename NAME openapi3 register delete NAME openapi3 register list [NAME] [--output=OUTPUT] openapi3 tbd openapi3 tbd merge [SERVICES...] [--dir=DIR] [--verbose] openapi3 tdb list [--dir=DIR] openapi3 tbd description [SERVICES...] [--dir=DIR] openapi3 tbd md FILE [--indent=INDENT] openapi3 tbd codegen [SERVICES...] [--srcdir=SRCDIR] [--destdir=DESTDIR] Arguments: DIR The directory of the specifications FILE The specification Options: --debug use the server in debug mode --verbose specifies to run in debug mode [default: False] --port=PORT the port for the server [default: 8080] --directory=DIRECTORY the directory in which the server is run --server=SERVER the server [default: flask] --output=OUTPUT the outputformat, table, csv, yaml, json [default: table] --srcdir=SRCDIR The directory of the specifications --destdir=DESTDIR The directory where the generated code should be put Description: This command does some useful things. """ map_parameters(arguments, 'fg', 'os', 'output', 'verbose', 'port', 'directory', 'yamldirectory', 'baseurl', 'filename', 'name') arguments.debug = arguments.verbose # VERBOSE(arguments) if arguments.generate: try: function = arguments.FUNCTION yamlfile = arguments.YAML baseurl = path_expand(arguments.baseurl) filename = arguments.filename.strip().split(".")[0] yamldirectory = path_expand(arguments.yamldirectory) sys.path.append(baseurl) module_name = pathlib.Path(f"{filename}").stem imported_module = import_module(module_name) func_obj = getattr(imported_module, function) setattr(sys.modules[module_name], function, func_obj) # get dataclasses defined in module dataclass_list = [] for attr_name in dir(imported_module): # # BUG: module is highloghted in pycharm # attr = getattr(imported_module, attr_name) if is_dataclass(attr): dataclass_list.append(attr) openAPI = generator.Generator() # BUG: this is windows specific and must be done differently # check if os.path.dirname, os.path.basename does this if sys.platform == 'win32': baseurl_short = baseurl.split("\\")[-1] else: baseurl_short = baseurl.split("/")[-1] openAPI.generate_openapi(func_obj, baseurl_short, yamldirectory, yamlfile, dataclass_list) except Exception as e: Console.error("Failed to generate openapi yaml") print(e) elif arguments.server and arguments.start and arguments.os: try: s = Server(spec=path_expand(arguments.YAML), directory=path_expand(arguments.directory) if arguments.directory else arguments.directory, port=arguments.port, server=arguments.wsgi, debug=arguments.debug, name=arguments.NAME) pid = s.run_os() VERBOSE(arguments, label="Server parameters") print(f"Run PID: {pid}") except FileNotFoundError: Console.error("specification file not found") except Exception as e: print(e) elif arguments.server and arguments.list: try: result = Server.list(self, name=arguments.NAME) # BUG: order= nt yet defined print(Printer.list(result)) except ConnectionError: Console.error("Server not running") elif arguments.server and arguments.ps: try: print() Console.info("Running Cloudmesh OpenAPI Servers") print() result = Server.ps(name=arguments.NAME) print(Printer.list(result, order=["name", "pid", "spec"])) print() except ConnectionError: Console.error("Server not running") elif arguments.server and arguments.stop and arguments.os: try: Server.stop(self, name=arguments.NAME) except ConnectionError: Console.error("Server not running") elif arguments.register and arguments.add: registry = Registry() result = registry.add(name=arguments.NAME, url=arguments.BASEURL, pid=arguments.PID) registry.Print(data=result, output=arguments.output) elif arguments.register and arguments.delete: registry = Registry() result = registry.delete(name=arguments.NAME) if result == 0: Console.error("Entry could not be found") else: Console.ok("Ok. Entry deleted") elif arguments.register and arguments.list: registry = Registry() result = registry.list(name=arguments.NAME) registry.Print(data=result, output=arguments.output) elif arguments.register and arguments.filename: registry = Registry() result = [registry.add_form_file(arguments.filename)] registry.Print(data=result, output=arguments.output) elif arguments.server and arguments.start: # VERBOSE(arguments) try: s = Server(name=arguments.NAME, spec=arguments.YAML, directory=arguments.directory, port=arguments.port, server=arguments.wsgi, debug=arguments.debug) print("spec: ", path_expand(arguments.YAML)) pid = s.start(name=arguments.NAME, spec=path_expand(arguments.YAML), foreground=arguments.fg) print(f"Run PID: {pid}") except FileNotFoundError: Console.error("specification file not found") except Exception as e: print(e) elif arguments.server and arguments.stop: try: print() Console.info("Stopping Cloudmesh OpenAPI Server") print() Server.stop(name=arguments.NAME) print() except ConnectionError: Console.error("Server not running") '''
def do_admin(self, args, arguments): """ :: Usage: admin mongo install [--brew] [--download=PATH] [--nosudo] [--docker] [--dryrun] [--force] admin mongo create admin mongo status admin mongo stats admin mongo version admin mongo start admin mongo stop admin mongo backup FILENAME admin mongo load FILENAME admin mongo security admin mongo password PASSWORD admin mongo list [--output=OUTPUT] admin mongo ssh admin mongo mode [MODE] admin status admin system info The admin command performs some administrative functions, such as installing packages, software and services. It also is used to start services and configure them. Arguments: FILENAME the filename for backups Options: -f specify the file Description: Mongo DB MongoDB is managed through a number of commands. The configuration is read from ~/.cloudmesh/cloudmesh.yaml First, you need to create a MongoDB database with cms admin mongo create Second, you need to start it with cms admin mongo start Now you can interact with it to find out the status, the stats, and the database listing with the commands cms admin mongo status cms admin mongo stats cms admin mongo list To stop it from running use the command cms admin mongo stop System information about your machine can be returned by cms admin system info This can be very useful in case you are filing an issue or bug. The command cms admin mongo ssh is only supported for docker and allows for debugging to login to the running container. This function may be disabled in future. admin mongo mode native switches configuration file to use native mode admin mongo mode running switches the configuration to use running mode """ map_parameters(arguments, "output", "nosudo", "docker", "dryrun", "force") arguments.output = arguments.output or "table" VERBOSE(arguments) # arguments.PATH = arguments['--download'] or None result = None if arguments.mongo: if arguments.install and arguments.docker: installer = MongoInstaller(dryrun=arguments.dryrun, force=arguments.force) r = installer.docker() return r elif arguments.install: print("MongoDB install") print(79 * "=") # print(arguments.force) installer = MongoInstaller(dryrun=arguments.dryrun, force=arguments.force) sudo = not arguments.nosudo # if 'linux' in platform.lower() : # print("SUDO:", sudo) # r = installer.install(sudo=sudo) r = installer.install() return r elif arguments.status: mongo = MongoDBController() state = mongo.status() if "error" in state["status"]: Console.error(state["message"]) print(Printer.attribute(state)) else: data = dotdict() for pid in state['output']: entry = state['output'][pid] data["pid"] = state['output'][pid] data["command"] = state['output'][pid][ 'command'].strip() print(Printer.dict(data, order=["pid", "command"])) Console.ok(str(data.pid['pid']) + " " + state["message"]) elif arguments.version: print("MongoDB Version") print(79 * "=") mongo = MongoDBController() r = mongo.version() print(r) elif arguments.security: mongo = MongoDBController() mongo.set_auth() print() elif arguments.create: print("MongoDB create") MongoDBController().create() elif arguments.ssh: print("MongoDB ssh") MongoDBController().ssh() elif arguments.start: print("MongoDB start") MongoDBController().start(security=True) elif arguments.stop: print("MongoDB stop") MongoDBController().stop() elif arguments.backup: print("MongoDB backup") MongoDBController().dump(arguments.get('FILENAME')) elif arguments.load: print("MongoDB backup") MongoDBController().restore(arguments.get('FILENAME')) elif arguments.stats: mongo = MongoDBController() r = mongo.stats() if len(r) > 0: print(Printer.attribute(r)) Console.ok("ok") else: Console.ok("is your MongoDB server running") elif arguments.list: mongo = MongoDBController() r = mongo.list() if len(r) > 0: if arguments.output == 'table': print(Printer.dict(r, order=["name", "sizeOnDisk", "empty", "collections"], output=arguments.output), ) else: print(Printer.write(r, output=arguments.output)) Console.ok("ok") else: Console.ok("is your MongoDB server running") elif arguments.mode: if arguments.MODE: if arguments.MODE not in ["native", "running", "docker"]: Console.error("The mode is not supported") config = Config() config["cloudmesh.data.mongo.MODE"] = arguments.MODE config.save() else: config = Config() mode = config["cloudmesh.data.mongo.MODE"] print(mode) return "" elif arguments.status: # config = Config() # data = config["cloudmesh.data.mongo"] print("Rest Service status") print("MongoDB status") try: mongo = MongoDBController() mongo.login() if mongo.status()['status'] == 'ok': Console.ok("Mongo is running") except Exception as e: Console.error("Mongo is not running") print(e) elif arguments.system: s = OperatingSystem.get() print(Printer.attribute(s)) return result
def do_storage(self, args, arguments): """ :: Usage: storage run storage monitor [--storage=SERVICES] [--status=all | --status=STATUS] [--output=output] [--clear] storage create dir DIRECTORY [--storage=SERVICE] [--parallel=N] storage get SOURCE DESTINATION [--recursive] [--storage=SERVICE] [--parallel=N] storage put SOURCE DESTINATION [--recursive] [--storage=SERVICE] [--parallel=N] storage list [SOURCE] [--recursive] [--parallel=N] [--output=OUTPUT] [--dryrun] storage delete SOURCE [--parallel=N] [--dryrun] storage search DIRECTORY FILENAME [--recursive] [--storage=SERVICE] [--parallel=N] [--output=OUTPUT] storage sync SOURCE DESTINATION [--name=NAME] [--async] [--storage=SERVICE] storage sync status [--name=NAME] [--storage=SERVICE] storage config list [--output=OUTPUT] storage copy --source=SOURCE:SOURCE_FILE_DIR --target=TARGET:TARGET_FILE_DIR This command does some useful things. Arguments: SOURCE SOURCE can be a directory or file DESTINATION DESTINATION can be a directory or file DIRECTORY DIRECTORY refers to a folder on the cloud service SOURCE:SOURCE_FILE_DIR source provider name: file or directory name TARGET:SOURCE_FILE_DIR destination provider name Options: --storage=SERVICE specify the cloud service name like aws or azure or box or google Description: commands used to upload, download, list files on different cloud storage services. storage run Execute the actions in database that are in waiting status. > storage monitor [--storage=SERVICE] > [--status=all | --status=STATUS] > [--output=output] > [--clear] Monitor the actions in database and refresh every 5 seconds. > storage put SOURCE DESTINATION [--recursive] [--storage=SERVICE] > [--parallel=N] Uploads the file specified in the filename to specified cloud from the SOURCEDIR. > storage get SOURCE DESTINATION [--recursive] [--storage=SERVICE] > [--parallel=N] Downloads the file specified in the filename from the specified cloud to the DESTDIR. storage delete SOURCE [--parallel=N] [--dryrun] Deletes the file specified in the filename from the specified cloud. > storage list [SOURCE] [--recursive] [--parallel=N] > [--output=OUTPUT] [--dryrun] lists all the files from the container name specified on the specified cloud. storage create dir DIRECTORY [--storage=SERVICE] [--parallel=N] creates a folder with the directory name specified on the specified cloud. > storage search DIRECTORY FILENAME [--recursive] > [--storage=SERVICE] > [--parallel=N] > [--output=OUTPUT] searches for the source in all the folders on the specified cloud. sync SOURCE DESTINATION puts the content of source to the destination. If --recursive is specified this is done recursively from the source If --async is specified, this is done asynchronously If a name is specified, the process can also be monitored with the status command by name. If the name is not specified all date is monitored. sync status The status for the asynchronous sync can be seen with this command config list Lists the configures storage services in the yaml file storage copy SOURCE DESTINATION Copies files from source storage to destination storage. The syntax of SOURCE and DESTINATION is: SOURCE - awss3:source.txt DESTINATION - azure:target.txt Description of the copy command: Command enables to Copy files between different cloud service providers, list and delete them. This command accepts `aws` , `google` and `local` as the SOURCE and TARGET provider. cms storage copy --source=SERVICE:SOURCE --target=DEST:TARGET Command copies files or directories from Source provider to Target Provider. cms storage slist --source=SERVICE:SOURCE Command lists all the files present in SOURCE provider's in the given SOURCE_FILE_DIR location This command accepts `aws` or `google` as the SOURCE provider cms storage sdelete --source=SERVICE:SOURCE Command deletes the file or directory from the SOURCE provider's SOURCE_FILE_DIR location Examples: cms storage_service copy --source=local:test1.txt --target=aws:uploadtest1.txt cms storage_service list --source=google:test cms storage_service delete --source=aws:uploadtest1.txt cms storage put test_file1.txt aws_test_file1.txt cms storage put ./recur_dir recur_dir_aws/ --recursive cms storage put ./recur_dir recur_dir_aws/ cms storage get aws_test_file1.txt aws_file1.txt cms storage get recur_dir_aws from_aws_dir cms storage get recur_dir_aws from_aws_dir --recursive cms storage list cms storage list --recursive cms storage list aws:recur_dir_aws --recursively cms storage delete aws:aws_test_file1.txt cms storage search recur_dir_aws recur_file1.txt Example: set storage=aws storage put SOURCE DESTINATION --recursive is the same as storage --storage=aws put SOURCE DESTINATION --recursive storage copy aws:source.txt oracle:target.txt """ # arguments.CONTAINER = arguments["--container"] VERBOSE(arguments) map_parameters(arguments, "dryrun", "recursive", "storage", "source", "target", "parallel") source = arguments.source target = arguments.target variables = Variables() parallelism = arguments.parallel or 1 arguments.storage = Parameter.expand(arguments.storage or variables['storage']) if arguments.monitor: provider = Provider(arguments.storage[0], parallelism=parallelism) status = arguments['--status'] or "all" output = arguments['--output'] or "table" result = provider.monitor(status=status, output=output) elif arguments.run: provider = Provider(arguments.storage[0], parallelism=parallelism) result = provider.run() elif arguments['get']: provider = Provider(arguments.storage[0], parallelism=parallelism) result = provider.get(arguments.SOURCE, arguments.DESTINATION, arguments.recursive) # result = provider.run() elif arguments.put: provider = Provider(arguments.storage[0], parallelism=parallelism) result = provider.put(arguments.SOURCE, arguments.DESTINATION, arguments.recursive) elif arguments.create and arguments.dir: provider = Provider(arguments.storage[0], parallelism=parallelism) result = provider.create_dir(arguments.DIRECTORY) elif arguments.list: """ storage list SOURCE [--parallel=N] """ if variables['storage']: default_source = f"{variables['storage']}:/" else: default_source = "local:/" sources = arguments.SOURCE or default_source sources = Parameter.expand(sources) deletes = [] for source in sources: storage, entry = Parameter.separate(source) storage = storage or source or "local" deletes.append((storage, entry)) _sources = ', '.join(sources) for delete in deletes: service, entry = delete if arguments.dryrun: print(f"Dryrun: list {service}:{entry}") else: provider = Provider(service=service, parallelism=parallelism) provider.list(name=entry, recursive=arguments.recursive) return "" elif arguments.delete: """ storage delete SOURCE [--parallel=N] """ if variables['storage']: default_source = f"{variables['storage']}:/" else: default_source = "local:/" sources = arguments.SOURCE or default_source sources = Parameter.expand(sources) deletes = [] for source in sources: storage, entry = Parameter.separate(source) storage = storage or source or "local" deletes.append((storage, entry)) _sources = ', '.join(sources) answer = yn_choice(f"Would you like to delete {_sources}?", default="no") if answer: for delete in deletes: service, entry = delete if arguments.dryrun: print(f"Dryrun: delete {service}:{entry}") else: provider = Provider(service=service, parallelism=parallelism) provider.delete(name=entry) else: Console.error("Deletion canceled") return "" elif arguments.search: for storage in arguments.storage: provider = Provider(storage, parallelism=parallelism) provider.search(arguments.DIRECTORY, arguments.FILENAME, arguments.recursive) elif arguments.rsync: # TODO: implement raise NotImplementedError elif arguments.copy: scloud, sfileDir = source.split(":", 1) or None tcloud, tfileDir = target.split(":", 1) or None print( f" Copying from Source {scloud} : {sfileDir} to Target {tcloud} : {tfileDir}" ) cloudName = ["aws", "google"] if scloud in cloudName: provider = Provider(service=scloud, parallelism=parallelism) provider.copyFiles(scloud, sfileDir, tcloud, tfileDir) else: print("Not Implemented") return ""
from cloudmesh.common.util import path_expand from cloudmesh.common.Benchmark import Benchmark from cloudmesh.management.configuration.name import Name from cloudmesh.common.util import HEADING from cloudmesh.common.console import Console from cloudmesh.configuration.Config import Config from cloudmesh.common.debug import VERBOSE import sys Benchmark.debug() config = Config() username = config["cloudmesh.profile.user"] if username == 'TBD': Console.error("please set cloudmesh.profile.user in ~/.cloudmesh.yaml") sys.exit() path = path_expand(f"{config.location}/name.yaml") data = { 'counter': 1, 'path': path, 'kind': "vm", 'schema': "{experiment}-{group}-{user}-{kind}-{counter}", 'experiment': 'exp', 'group': 'group', 'user': '******' } try: os.remove(path)
def update(self, _entries, progress=True): MongoDBController().start_if_not_running() if type(_entries) == dict: entries = [_entries] else: entries = _entries if progress: bar = Bar('Cloudmesh Database Update', max=len(entries)) result = [] for entry in entries: if progress: bar.next() if 'cm' not in entry: print("UPDATE ERROR") VERBOSE(entry) raise ValueError("The cm attribute is not in the entry") entry['cm']['collection'] = "{cloud}-{kind}".format(**entry["cm"]) # noinspection PyUnusedLocal try: self.col = self.db[entry['cm']['collection']] old_entry = self.col.find_one({ "cm.kind": entry["cm"]["kind"], "cm.cloud": entry["cm"]["cloud"], "cm.name": entry["cm"]["name"] }) if old_entry is not None: cm = dict(old_entry['cm']) cm.update(entry['cm']) cm['modified'] = str(datetime.utcnow()) # entry['cm']['created'] = cm['created'] entry['cm'] = cm post = self.col.replace_one( { "cm.kind": entry['cm']["kind"], "cm.cloud": entry['cm']["cloud"], "cm.name": entry['cm']["name"] }, entry, upsert=True) else: entry['cm']['created'] = entry['cm']['modified'] = str( datetime.utcnow()) self.col.insert_one(entry) except Exception as e: Console.error( "uploading document\n{entry}\n-------\n\n".format( entry=str(entry))) pass result.append(entry) if progress: bar.finish() return result
def do_test(self, args, arguments): """ :: Usage: test This command is intended to check if your windows set up is correctly done. Bugs: This program is supposed to be implemented. It is at this time just a template Description: Checks we do 1. are you running python 3.8.1 2. are you having the newest version of pip 3. is cl installed 4. is nmake installed 5. is the username without spaces 6. are you running in a vnenv 7. is the default mongo port used 8. do you have docker installed 9. do you have vbox installed 10. how much memory do you have 11. do you have free diskspace Checks that are missing or need implemented 12. is hyperv switched on or off 13. are containers running 14. .... other tyings that can help us debug your environment """ tester = CloudmeshTest() # # Python setup # tester.check_venv() tester.check_python() tester.check_command("pip --version", test="20.0.2") # # command tool setup # if platform.system() == "Windows": if not tester.check_windows(): Console.error(" THIS VERSION OF WINDOWS IS NOT SUPPORTED.") return "" tester.which("cl") tester.which("nmake") tester.which("git") tester.which("ssh") tester.which("ssh-keygen") tester.which("docker") tester.which("yamllint") else: tester.check_command("git --version", test="git version") tester.check_command("ssh", test="usage", show=False) tester.check_command("ssh-keygen --help", test="usage", show=False) tester.check_command("docker --version", test="Docker version") tester.check_command("VirtualBox --help", test="Oracle VM VirtualBox VM Selector", show=False) tester.check_command("yamllint", test="usage: yamllint", show=False) tester.is_user_name_valid() tester.check_mongo() tester.usage() tester.check_yaml() return ""
def ssh(self, vm=None, command=None): def key_selector(keys): """ This is a helper method for ssh key selection THIS IS JUST A SAFETY MEASURE, PLEASE DON'T MIND IT :param keys: :return: """ tmp_keys = keys[:] # indices = range(1,len(tmp_keys)+1) for key_idx, key in enumerate(keys): key['idx'] = key_idx + 1 print( Printer.flatwrite( tmp_keys, sort_keys=["idx"], order=['idx', 'KeyName', 'KeyFingerprint'], header=['Index', 'Key Name', "Key Fingerprint"], output="table", humanize=None)) # Console.msg("Please select one of the AWS key indices from the table above: ") picked = 0 while picked < 1 or picked > len(keys): try: picked = int( input( "Please select one of the AWS key indices from the table above: " )) except ValueError: pass return keys[picked - 1] cm = CmDatabase() ip = vm['public_ips'] try: key_name = vm['KeyName'] keys = cm.find_all_by_name(name=key_name, kind="key") for k in keys: if 'location' in k.keys(): if 'private' in k['location'].keys(): key = k['location']['private'] break except (KeyError, IndexError): aws_keys = cm.find(kind='key', cloud='aws') if len(aws_keys) == 0: Console.error( f"Could not find a key for the AWS instance '{vm['name']}'" ) Console.error( f"Use `cms help key` to learn how to add and upload a key for AWS" ) return aws_key = key_selector(aws_keys) for sshkey in cm.find_all_by_name(name=aws_key['KeyName'], kind="key"): if "location" in sshkey.keys(): key = sshkey['location']['private'] break user = "******" # needs to be set on creation. if command is None: command = "" if user is None: location = ip else: location = user + '@' + ip cmd = "ssh " \ "-o StrictHostKeyChecking=no " \ "-o UserKnownHostsFile=/dev/null " \ f"-i {key} {location} {command}" cmd = cmd.strip() print(cmd) # VERBOSE(cmd) if command == "": if platform.lower() == 'win32': class disable_file_system_redirection: _disable = ctypes.windll.kernel32.Wow64DisableWow64FsRedirection _revert = ctypes.windll.kernel32.Wow64RevertWow64FsRedirection def __enter__(self): self.old_value = ctypes.c_long() self.success = self._disable( ctypes.byref(self.old_value)) def __exit__(self, type, value, traceback): if self.success: self._revert(self.old_value) with disable_file_system_redirection(): os.system(cmd) else: os.system(cmd) else: if platform.lower() == 'win32': class disable_file_system_redirection: _disable = ctypes.windll.kernel32.Wow64DisableWow64FsRedirection _revert = ctypes.windll.kernel32.Wow64RevertWow64FsRedirection def __enter__(self): self.old_value = ctypes.c_long() self.success = self._disable( ctypes.byref(self.old_value)) def __exit__(self, type, value, traceback): if self.success: self._revert(self.old_value) with disable_file_system_redirection(): ssh = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) else: ssh = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) result = ssh.stdout.read().decode("utf-8") if not result: error = ssh.stderr.readlines() print("ERROR: %s" % error) else: return result
def copy(self, source=None, source_obj=None, target=None, target_obj=None, recursive=True): """ Copy objects from source to target storage :param source: source CSP - awss3/azure/local :param source_obj: It can be file or folder :param target: target CSP - awss3/azure/local :param target_obj: It can be file or folder :param recursive: enlist directories/sub-directories :return: dictionary enlisting copied objects """ banner(f"CALLING AZURE BLOB STORAGE PROVIDER'S GET METHOD FOR " f"{source.upper()} TO {target.upper()} COPY") if target_obj is None: target_obj = source_obj target_obj = target_obj.replace("\\", "/") source_obj = source_obj.replace("\\", "/") if target == "local": result = self.storage_provider.get(source=source_obj, destination=target_obj, recursive=recursive) elif target == "azure": source_obj = str(Path(source_obj).expanduser()).replace("\\", "/") if source == "awss3": source_provider = StorageAwss3Provider(service='awss3') config = Config(config_path="~/.cloudmesh/cloudmesh.yaml") spec = config["cloudmesh.storage"] local_target = spec["local"]["default"]["directory"] local_target = local_target.replace("\\", "/") result = source_provider.get(source=source_obj, destination=local_target, recursive=recursive) print("Fetched from s3 to local:\n") # pprint(result) # TODO: return error if get fails, no put required source_obj = Path(Path(local_target).expanduser() / source_obj) result = self.storage_provider.put(source=source_obj, destination=target_obj, recursive=recursive) else: raise NotImplementedError if result is None: return Console.error(f"Object {source_obj} couldn't be copied " f"from {source} to {target}. Please check.") else: Console.ok( f"Copied {source_obj} from {source} to {target}\nTarget " f"object name is {target_obj} ") pprint(result) return self.print_table(result, status="Copied", source=source, target=target)
def create(self, name=None, image=None, size=None, location=None, timeout=360, key=None, secgroup=None, ip=None, user=None, public=None, group=None, metadata=None, **kwargs): """ creates a named node :param name: the name of the node :param image: the image used :param size: the size of the image :param timeout: a timeout in seconds that is invoked in case the image does not boot. The default is set to 3 minutes. :param kwargs: additional arguments passed along at time of boot :return: the list with the modified dicts """ """ create one node """ if not ip and public: ip = self.find_available_public_ip() elif ip is not None: entry = self.list_public_ips(ip=ip, available=True) if len(entry) == 0: Console.error("ip not available") return None banner("Create Server") Console.msg(f" Name: {name}") Console.msg(f" User: {user}") Console.msg(f" IP: {ip}") Console.msg(f" Image: {image}") Console.msg(f" Size: {size}") Console.msg(f" Public: {public}") Console.msg(f" Key: {key}") Console.msg(f" Location: {location}") Console.msg(f" Timeout: {timeout}") Console.msg(f" Secgroup: {secgroup}") Console.msg(f" Group: {group}") # Console.msg(f" Groups: {groups}") Console.msg("") if secgroup is None: secgroup = 'default' if key is None: raise ValueError("Key must be set. Use cms set key=<key name>") # # BUG: the tags seem incomplete # if metadata is None: metadata = [] metadata = [{ 'Key': 'cm.image', 'Value': image }, { 'Key': 'cm.name', 'Value': name }, { 'Key': 'cm.flavor', 'Value': size }, { 'Key': 'cm.user', 'Value': self.user }, { 'Key': 'cm.kind', 'Value': "vm" }, { 'Key': 'cm.status', 'Value': "BOOTING" }, { 'Key': 'Name', 'Value': name }] # VERBOSE(metadata) new_ec2_instance = self.ec2_resource.create_instances( ImageId=image, InstanceType=size, MaxCount=1, MinCount=1, SecurityGroups=[secgroup], KeyName=key, TagSpecifications=[{ 'ResourceType': 'instance', 'Tags': metadata }]) # VERBOSE(new_ec2_instance) new_ec2_instance = new_ec2_instance[0] waiter = self.ec2_client.get_waiter('instance_exists') waiter.wait(Filters=[{ 'Name': 'instance-id', 'Values': [new_ec2_instance.instance_id] }], WaiterConfig={ 'Delay': 20, 'MaxAttempts': timeout / 20 }) print() Console.ok("Instance created...") print() # if IP provided, Attach it to new instance if ip: self.attach_public_ip(name, ip) # x = self.ec2_client.describe_instances(InstanceIds=[new_ec2_instance.instance_id]) # VERBOSE(x) data = self.info(name=name) # VERBOSE(data) data['name'] = name data['kind'] = 'aws' data['status'] = new_ec2_instance.state['Name'], data['created'] = new_ec2_instance.launch_time.strftime( "%m/%d/%Y, %H:%M:%S") if new_ec2_instance.launch_time else '', data['updated'] = new_ec2_instance.launch_time.strftime( "%m/%d/%Y, %H:%M:%S") if new_ec2_instance.launch_time else '', data['name'] = new_ec2_instance.tags[0][ 'Value'] if new_ec2_instance.tags else '', data['instance_id'] = new_ec2_instance.id, data['image'] = new_ec2_instance.image_id, data['key_name'] = key, Console.msg("Waiting for the Public IP address assignment ...") while True: try: public_ip = \ self.ec2_client.describe_instances( InstanceIds=[new_ec2_instance.id])['Reservations'][0][ 'Instances'] \ [0]['PublicIpAddress'], break except KeyError: time.sleep(0.5) data['public_ips'] = public_ip[0] data['private_ips'] = new_ec2_instance.private_ip_address Console.msg(f" Public IP: {data['public_ips']}") Console.msg(f" Private IP: {data['private_ips']}") output = self.update_dict(data, kind="vm")[0] return output
def __init__(self, name=None, configuration="~/.cloudmesh/cloudmesh.yaml"): """ Initializes the provider. The default parameters are read from the configuration file that is defined in yaml format. :param name: The name of the provider as defined in the yaml file :param configuration: The location of the yaml configuration file """ conf = Config(configuration)["cloudmesh"] self.user = Config()["cloudmesh"]["profile"]["user"] self.spec = conf["cloud"][name] self.cloud = name cred = self.spec["credentials"] self.default = self.spec["default"] self.cloudtype = self.spec["cm"]["kind"] super().__init__(name, conf) VERBOSE(cred, verbose=10) if self.cloudtype != 'azure': Console.error("This class is meant for azure cloud") # ServicePrincipalCredentials related Variables to configure in # cloudmesh.yaml file # AZURE_APPLICATION_ID = '<Application ID from Azure Active Directory # App Registration Process>' # AZURE_SECRET_KEY = '<Secret Key from Application configured in # Azure>' # AZURE_TENANT_ID = '<Directory ID from Azure Active Directory # section>' credentials = ServicePrincipalCredentials( client_id=cred['AZURE_APPLICATION_ID'], secret=cred['AZURE_SECRET_KEY'], tenant=cred['AZURE_TENANT_ID'] ) subscription = cred['AZURE_SUBSCRIPTION_ID'] # Management Clients self.resource_client = ResourceManagementClient( credentials, subscription) self.compute_client = ComputeManagementClient( credentials, subscription) self.network_client = NetworkManagementClient( credentials, subscription) # VMs abbreviation self.vms = self.compute_client.virtual_machines self.imgs = self.compute_client.virtual_machine_images # Azure Resource Group self.GROUP_NAME = self.default["resource_group"] # Azure Datacenter Region self.LOCATION = cred["AZURE_REGION"] # NetworkManagementClient related Variables self.VNET_NAME = self.default["network"] self.SUBNET_NAME = self.default["subnet"] self.IP_CONFIG_NAME = self.default["AZURE_VM_IP_CONFIG"] self.NIC_NAME = self.default["AZURE_VM_NIC"] # Azure VM Storage details self.OS_DISK_NAME = self.default["AZURE_VM_DISK_NAME"] self.USERNAME = self.default["AZURE_VM_USER"] self.PASSWORD = self.default["AZURE_VM_PASSWORD"] self.VM_NAME = self.default["AZURE_VM_NAME"] # Create or Update Resource group self.get_resource_group()
def do_vm(self, args, arguments): """ :: Usage: vm ping [NAMES] [--cloud=CLOUDS] [--count=N] vm check [NAMES] [--cloud=CLOUDS] [--username=USERNAME] vm status [NAMES] [--cloud=CLOUDS] [--output=OUTPUT] vm console [NAME] [--force] vm log [NAME] [--force] vm stop [NAMES] [--dryrun] vm start [NAMES] [--dryrun] vm terminate [NAMES] [--cloud=CLOUD] [--dryrun] vm delete [NAMES] [--cloud=CLOUD] [--dryrun] vm refresh [--cloud=CLOUDS] vm list [NAMES] [--cloud=CLOUDS] [--output=OUTPUT] [--refresh] vm boot [--n=COUNT] [--name=VMNAMES] [--cloud=CLOUD] [--username=USERNAME] [--image=IMAGE] [--flavor=FLAVOR] [--public] [--secgroup=SECGROUPs] [--group=GROUPs] [--key=KEY] [--dryrun] [-v] vm meta list [NAME] vm meta set [NAME] KEY=VALUE... vm meta delete [NAME] KEY... vm script [--name=NAMES] [--username=USERNAME] [--key=KEY] [--dryrun] [--dir=DESTINATION] SCRIPT vm ip assign [NAMES] [--cloud=CLOUD] vm ip show [NAMES] [--group=GROUP] [--cloud=CLOUD] [--output=OUTPUT] [--refresh] vm ip inventory [NAMES] vm ssh [NAMES] [--username=USER] [--quiet] [--ip=IP] [--key=KEY] [--command=COMMAND] vm put SOURCE DESTINATION [NAMES] vm get SOURCE DESTINATION [NAMES] vm rename [OLDNAMES] [NEWNAMES] [--force] [--dryrun] vm wait [--cloud=CLOUD] [--interval=INTERVAL] [--timeout=TIMEOUT] vm info [--cloud=CLOUD] [--output=OUTPUT] vm username USERNAME [NAMES] [--cloud=CLOUD] vm resize [NAMES] [--size=SIZE] Arguments: OUTPUT the output format COMMAND positional arguments, the commands you want to execute on the server(e.g. ls -a) separated by ';', you will get a return of executing result instead of login to the server, note that type in -- is suggested before you input the commands NAME server name. By default it is set to the name of last vm from database. NAMES server name. By default it is set to the name of last vm from database. KEYPAIR_NAME Name of the vm keypair to be used to create VM. Note this is not a path to key. NEWNAMES New names of the VM while renaming. OLDNAMES Old names of the VM while renaming. Options: -v verbose, prints the dict at the end --output=OUTPUT the output format -H --modify-knownhosts Do not modify ~/.ssh/known_hosts file when ssh'ing into a machine --username=USERNAME the username to login into the vm. If not specified it will be guessed from the image name and the cloud --ip=IP give the public ip of the server --cloud=CLOUD give a cloud to work on, if not given, selected or default cloud will be used --count=COUNT give the number of servers to start --detail for table, a brief version is used as default, use this flag to print detailed table --flavor=FLAVOR give the name or id of the flavor --group=GROUP give the group name of server --secgroup=SECGROUP security group name for the server --image=IMAGE give the name or id of the image --key=KEY specify a key to use, input a string which is the full path to the private key file --keypair_name=KEYPAIR_NAME Name of the vm keypair to be used to create VM. Note this is not a path to key. --user=USER give the user name of the server that you want to use to login --name=NAME give the name of the virtual machine --force rename/ delete vms without user's confirmation --command=COMMAND specify the commands to be executed Description: commands used to boot, start or delete servers of a cloud vm default [options...] Displays default parameters that are set for vm boot either on the default cloud or the specified cloud. vm boot [options...] Boots servers on a cloud, user may specify flavor, image .etc, otherwise default values will be used, see how to set default values of a cloud: cloud help vm start [options...] Starts a suspended or stopped vm instance. vm stop [options...] Stops a vm instance . vm delete [options...] Delete servers of a cloud, user may delete a server by its name or id, delete servers of a group or servers of a cloud, give prefix and/or range to find servers by their names. Or user may specify more options to narrow the search vm floating_ip_assign [options...] assign a public ip to a VM of a cloud vm ip show [options...] show the ips of VMs vm ssh [options...] login to a server or execute commands on it vm list [options...] same as command "list vm", please refer to it vm status [options...] Retrieves status of last VM booted on cloud and displays it. vm refresh [--cloud=CLOUDS] this command refreshes the data for virtual machines, images and flavors for the specified clouds. vm ping [NAMES] [--cloud=CLOUDS] [--count=N] [--processors=PROCESSORS] pings the specified virtual machines, while using at most N pings. The ping is executed in parallel. If names are specifies the ping is restricted to the given names in parameter format. If clouds are specified, names that are not in these clouds are ignored. If the name is set in the variables this name is used. cms vm ssh --command=\"uname -a\" executes the uname command on the last booted vm vm script [--name=NAMES] [--username=USERNAME] [--key=KEY] [--dryrun] [--dir=DESTINATION] [--shell=SHELL] SCRIPT The script command copies a shell script to the specified vms into the DESTINATION directory and than execute it. With SHELL you can set the shell for executing the command, this coudl even be a python interpreter. Examples for SHELL are /bin/sh, /usr/bin/env python vm put SOURCE DESTINATION [NAMES] puts the file defined by SOURCE into the DESINATION folder on the specified machines. If the file exists it is overwritten, so be careful. vm get SOURCE DESTINATION [NAMES] gets the file defined by SOURCE into the DESINATION folder on the specified machines. The SOURCE is on the remote machine. If one machine is specified, the SOURCE is the same name as on the remote machine. If multiple machines are specified, the name of the machine will be a prefix to the filename. If the filenames exists, they will be overwritten, so be careful. Tip: give the VM name, but in a hostlist style, which is very convenient when you need a range of VMs e.g. sample[1-3] => ['sample1', 'sample2', 'sample3'] sample[1-3,18] => ['sample1', 'sample2', 'sample3', 'sample18'] Quoting commands: cm vm login gregor-004 --command=\"uname -a\" Limitations: Azure: rename is not supported """ map_parameters(arguments, 'active', 'cloud', 'command', 'dryrun', 'flavor', 'force', 'group' 'output', 'group', 'image', 'interval', 'timeout', 'ip', 'key', 'modify-knownhosts', 'n', 'name', 'public', 'quiet', 'secgroup', 'size', 'username', 'output', 'count', 'refresh') variables = Variables() database = CmDatabase() arguments.output = Parameter.find("output", arguments, variables, "table") arguments.refresh = Parameter.find_bool("refresh", arguments, variables) if (arguments.meta and arguments.list): name = arguments.NAME if arguments.NAME is None: name = variables['vm'] if name is None: Console.error("No vm specified") cloud = "chameleon" # cloud = Parameter.find(arguments, variables) print(f"vm metadata for {name} on {cloud}") provider = Provider(name=cloud) r = provider.get_server_metadata(name) print(r) elif arguments.meta and arguments.set: metadata = {} pairs = arguments['KEY=VALUE'] for pair in pairs: key, value = pair.split("=", 1) metadata[key] = value name = arguments.NAME if arguments.NAME is None: name = variables['vm'] if name is None: Console.error("No vm specified") cloud = "chameleon" # cloud = Parameter.find(arguments, variables) print(f"cloud {cloud} {name}") provider = Provider(name=cloud) provider.set_server_metadata(name, **metadata) r = provider.get_server_metadata(name) pprint(r) elif arguments.meta and arguments.delete: metadata = {} keys = arguments['KEY'] name = arguments.NAME if arguments.NAME is None: name = variables['vm'] if name is None: Console.error("No vm specified") cloud = "chameleon" # cloud = Parameter.find(arguments, variables) print(f"cloud {cloud} {name}") provider = Provider(name=cloud) for key in keys: provider.delete_server_metadata(name, key) r = provider.get_server_metadata(name) pprint(r) elif arguments.list and arguments.refresh: names = [] clouds, names = Arguments.get_cloud_and_names("list", arguments, variables) for cloud in clouds: print(f"cloud {cloud}") provider = Provider(name=cloud) vms = provider.list() provider.Print(vms, output=arguments.output, kind="vm") elif arguments.list: names = [] clouds, names = Arguments.get_cloud_and_names("list", arguments, variables) try: for cloud in clouds: print(f"List {cloud}") p = Provider(cloud) kind = p.kind collection = "{cloud}-vm".format(cloud=cloud, kind=p.kind) db = CmDatabase() vms = db.find(collection=collection) p.Print(vms, output=arguments.output, kind="vm") except Exception as e: VERBOSE(e) return "" elif arguments.ping: raise NotImplementedError """ vm ping [NAMES] [--cloud=CLOUDS] [--count=N] """ if arguments.NAMES: variables['vm'] = arguments.NAMES if arguments['--cloud']: variables['cloud'] = arguments['--cloud'] clouds, names = Arguments.get_cloud_and_names("status", arguments, variables) count = arguments.count if arguments.count: count = int(count) else: count = 1 ips = set() for cloud in clouds: params = {} # gets public ips from database cursor = database.db[f'{cloud}-vm'] for name in names: for node in cursor.find({'name': name}): ips.update(set(node['public_ips'])) ips = list(ips) pprint(ips) for ip in ips: Shell.ping(host=ip, count=count) elif arguments.check: raise NotImplementedError """ vm check [NAMES] [--cloud=CLOUDS] [--username=USERNAME] """ """ THIS IS ALL WRONG AS PROVIDER DEPENDENT !!! if arguments.NAMES: variables['vm'] = arguments.NAMES if arguments['--cloud']: variables['cloud'] = arguments['--cloud'] clouds, names = Arguments.get_cloud_and_names("status", arguments, variables) for cloud in clouds: provider = Provider(cloud) params = {} params['key'] = \ provider.p.spec["credentials"]['EC2_PRIVATE_KEY_FILE_PATH'] + \ provider.p.spec["credentials"]['EC2_PRIVATE_KEY_FILE_NAME'] params['username'] = arguments['--username'] # or get from db processors = arguments['--processors'] if processors: params['processors'] = int(processors[0]) # gets public ips from database public_ips = [] cursor = database.db['{cloud}-vm'] for name in names: for node in cursor.find({'name': name}): public_ips.append(node['public_ips']) public_ips = [y for x in public_ips for y in x] Host.check(hosts=public_ips, **params) """ elif arguments.status: if arguments.NAMES: variables['vm'] = arguments.NAMES if arguments['--cloud']: variables['cloud'] = arguments['--cloud'] clouds, names = Arguments.get_cloud_and_names("status", arguments, variables) # gets status from database for cloud in clouds: provider = Provider(cloud) status = [] cursor = database.db[f'{cloud}-vm'] print(cloud) for name in names: for node in cursor.find({'name': name}): status.append(node) provider.Print(status, output=arguments.output, kind="status") return "" elif arguments.start: # TODO: not tested if arguments.NAMES: names = variables['vm'] = arguments.NAMES if arguments['--cloud']: variables['cloud'] = arguments['--cloud'] clouds, names = Arguments.get_cloud_and_names("stop", arguments, variables) cloud = clouds[0] print(cloud) print(names) for name in names: provider = Provider(cloud) if arguments['--dryrun']: print(f"start node {name}") else: vms = provider.start(names=name, cloud=cloud) provider.Print(vms, output=arguments.output, kind="vm") return "" elif arguments.stop: # TODO: not tested if arguments.NAMES: variables['vm'] = arguments.NAMES if arguments['--cloud']: variables['cloud'] = arguments['--cloud'] clouds, names = Arguments.get_cloud_and_names("stop", arguments, variables) for cloud in clouds: params = {} provider = Provider(cloud) if arguments['--dryrun']: Console.ok(f"Dryrun stop: " f" {cloud}\n" f" {names}" f" {provider}") else: for name in names: vms = provider.stop(name) provider.Print(vms, output=arguments.output, kind="vm") elif arguments.terminate: # TODO: not tested if arguments.NAMES: variables['vm'] = arguments.NAMES if arguments['--cloud']: variables['cloud'] = arguments['--cloud'] clouds, names = Arguments.get_cloud_and_names("stop", arguments, variables) for cloud in clouds: params = {} provider = Provider(cloud) if arguments['--dryrun']: Console.ok(f"Dryrun terminate: " f" {cloud}\n" f" {names}" f" {provider}") else: for name in names: vms = provider.destroy(name) provider.Print(vms, output=arguments.output, kind="vm") elif arguments.delete: if arguments.NAMES: variables['vm'] = arguments.NAMES if arguments['--cloud']: variables['cloud'] = arguments['--cloud'] clouds, names = Arguments.get_cloud_and_names("stop", arguments, variables) if names is not None: pass elif clouds is not None: for cloud in clouds: provider = Provider(cloud) vms = provider.list() for vm in vms: r = provider.destroy(name=vm) return "" else: return "" for cloud in clouds: provider = Provider(cloud) vms = provider.list() for vm in vms: name = vm["cm"]["name"] if name in names: r = provider.destroy(name=name) # TODO: username, secgroup elif arguments.boot: # not everything works """ vm boot [--name=VMNAMES] [--cloud=CLOUD] [--username=USERNAME] [--image=IMAGE] [--flavor=FLAVOR] [--public] [--secgroup=SECGROUP] [--key=KEY] [--group=GROUP] [--dryrun] """ # for name in names: # node = p.create(name=name, size=flavor, image=image) # VERBOSE(arguments) parameters = dotdict() names = Parameter.expand(arguments.name) cloud = Parameter.find("cloud", arguments, variables.dict()) defaults = Config()[f"cloudmesh.cloud.{cloud}.default"] groups = Parameter.find("group", arguments, variables.dict(), {"group": "default"}) parameters = dotdict() parameters.names = arguments.name parameters.group = groups for attribute in ["image", "username", "flavor", "key", "secgroup"]: parameters[attribute] = Parameter.find(attribute, arguments, variables.dict(), defaults) if arguments.username is None: parameters.user = Image.guess_username(parameters.image) provider = Provider(name=cloud) parameters.secgroup = arguments.secgroup or "default" # pprint(parameters) if arguments['--dryrun']: Console.ok(f"Dryrun stop: \n" f" cloud={cloud}\n" f" names={names}\n" f" provide={provider}") for attribute in parameters: value = parameters[attribute] Console.ok(f" {attribute}={value}") else: # pprint (parameters) if not arguments.n: count = 1 else: count = int(arguments.n) for i in range(0, count): if names is None: n = Name() n.incr() parameters.names = str(n) # parameters.progress = len(parameters.names) < 2 vms = provider.create(**parameters) variables['vm'] = str(n) if arguments["-v"]: banner("Details") pprint(vms) # provider.Print(arguments.output, "vm", vms) elif arguments.info: """ vm info [--cloud=CLOUD] [--output=OUTPUT] """ print("info for the vm") cloud, names = Arguments.get_cloud_and_names("info", arguments, variables) raise NotImplementedError elif arguments.rename: raise NotImplementedError # Not tested print("rename the vm") v = Variables() cloud = v["cloud"] p = Provider(cloud) try: oldnames = Parameter.expand(arguments["OLDNAMES"]) newnames = Parameter.expand(arguments["NEWNAMES"]) force = arguments["--force"] if oldnames is None or newnames is None: Console.error("Wrong VMs specified for rename", traceflag=False) elif len(oldnames) != len(newnames): Console.error("The number of VMs to be renamed is wrong", traceflag=False) else: print(oldnames) print(newnames) for i in range(0, len(oldnames)): oldname = oldnames[i] newname = newnames[i] if arguments["--dryrun"]: Console.ok( "Rename {} to {}".format(oldname, newname)) else: print(f"rename {oldname} -> {newname}") p.rename(source=oldname, destination=newname) msg = "info. OK." Console.ok(msg) except Exception as e: Error.traceback(e) Console.error("Problem renaming instances", traceflag=True) elif arguments["ip"] and arguments["show"]: raise NotImplementedError print("show the ips") """ vm ip show [NAMES] [--group=GROUP] [--cloud=CLOUD] [--output=OUTPUT] [--refresh] """ elif arguments["ip"] and arguments["assign"]: raise NotImplementedError """ vm ip assign [NAMES] [--cloud=CLOUD] """ print("assign the public ip") elif arguments["ip"] and arguments["inventory"]: raise NotImplementedError """ vm ip inventory [NAMES] """ print("list ips that could be assigned") elif arguments.default: raise NotImplementedError print("sets defaults for the vm") elif arguments.script: raise NotImplementedError clouds, names = Arguments.get_cloud_and_names("run", arguments, variables) username = arguments['--username'] script = arguments.SCRIPT for cloud in clouds: provider = Provider(cloud) name_ips = {} cursor = database.db['{}-node'.format(cloud)] for name in names: for node in cursor.find({'name': name}): name_ips[name] = node['public_ips'] if arguments['--dryrun']: print("run script {} on vms: {}".format(script, names)) else: provider.ssh(name_ips, username=username, script=script) elif arguments.username: raise NotImplementedError """ vm username USERNAME [NAMES] [--cloud=CLOUD] """ print("sets the username for the vm") elif arguments.resize: raise NotImplementedError """ vm resize [NAMES] [--size=SIZE] """ pass elif arguments.ssh: """ vm ssh [NAMES] [--username=USER] [--quiet] [--ip=IP] [--key=KEY] [--command=COMMAND] """ # VERBOSE(arguments) clouds, names, command = Arguments.get_commands("ssh", arguments, variables) # print (clouds) # print(names) # print (command) if arguments.command is None and len(names) > 1: Console.error("Interactive shell can only be done on one vm") return "" elif arguments.command is None and len(names) == 1: name = names[0] cloud = clouds[0] cm = CmDatabase() try: vm = cm.find_name(name, "vm")[0] except IndexError: Console.error(f"could not find vm {name}") return "" # VERBOSE(vm) cloud = vm["cm"]["cloud"] provider = Provider(name=cloud) provider.ssh(vm=vm) return "" else: # command on all vms if clouds is None or names is None or command is None: return "" else: for cloud in clouds: p = Provider(cloud) for name in names: cm = CmDatabase() try: vm = cm.find_name(name, "vm")[0] except IndexError: Console.error(f"could not find vm {name}") continue r = p.ssh(vm=vm, command=command) print(r) return "" elif arguments.console: # why is this not vm clouds, names, command = Arguments.get_commands("ssh", arguments, variables) print(clouds) print(names) print(command) for cloud in clouds: p = Provider(cloud) for name in names: cm = CmDatabase() try: vm = cm.find_name(name, "vm")[0] except IndexError: Console.error(f"could not find vm {name}") continue r = p.console(vm=vm) print(r) return "" elif arguments.log: # why is this not vm clouds, names, command = Arguments.get_commands("ssh", arguments, variables) print(clouds) print(names) print(command) for cloud in clouds: p = Provider(cloud) for name in names: cm = CmDatabase() try: vm = cm.find_name(name, "vm")[0] except IndexError: Console.error(f"could not find vm {name}") continue r = p.log(vm=vm) print(r) return "" elif arguments.wait: """ vm wait [--cloud=CLOUD] [--interval=INTERVAL] [--timeout=TIMEOUT] """ # why is this not vm clouds, names, command = Arguments.get_commands("ssh", arguments, variables) # print (clouds) # print (names) # print (command) for cloud in clouds: p = Provider(cloud) for name in names: cm = CmDatabase() try: vm = cm.find_name(name, "vm")[0] except IndexError: Console.error(f"could not find vm {name}") continue r = p.wait(vm=vm, interval=arguments.interval, timeout=arguments.timeout) if r: Console.ok("Instance available for SSH") else: Console.error( f"Instance unavailable after timeout of {arguments.timeout}") # print(r) return ""
def do_queue(self, args, arguments): """ :: Usage: queue create --name=NAME --policy=POLICY --cluster=CLUSTER [--charge=CHARGE] [--unit=UNIT] queue activate [--name=NAME] queue deactivate [--name=NAME] queue set unit queue connection_test --job=JOB queue cluster list [--cluster=CLUSTERS] [--depth=DEPTH] queue cluster remove [--cluster=CLUSTERS] queue cluster set [--cluster=CLUSTERS] PARAMETER=VALUE Arguments: FILE a file name INPUT_TYPE tbd Options: -f specify the file --depth=DEPTH [default: 1] --format=FORMAT [default: table] Description: This command creates a queue that is associated with a cloud. We assume that a number of experiments are conducted with possibly running the script multiple times. Each experiment will save the batch script in its own folder. The output of the script can be saved in a destination folder. A virtual directory is used to coordinate all saved files. The files can be located due to the use of the virtual directory on multiple different data or file services Authentication to the Batch systems is done viw the underlaying HPC center authentication. We assume that the user has an account to submit on these systems. (SSH, 2 factor, XSEDE-account) TBD. Examples: LOTS OF DOCUMENTATION MISSING HERE [--companion-file=COMPANION_FILE] [--outfile-name=OUTPUT_FILE_NAME] [--suffix=SUFFIX] [--overwrite] """ # # create slurm manager so it can be used in all commands # queue = Queue() # debug=arguments["--debug"]) # arguments["--cloud"] = "test" # arguments["NAME"] = "fix" # map_parameters(arguments, # "cloud", # "name", # "cluster", # "script", # "type", # "destination", # "source", # "format") # if not arguments.create # find cluster name from Variables() # if no cluster is defined look it up in yaml in batch default: # if not defined there fail # clusters = Parameter.expand(arguments.cluster) # name = Parameters.expand[argumnets.name) # this will return an array of clusters and names of jobs and all cluster # job or clusterc commands will be executed on them # see the vm # # if active: False in the yaml file for the cluster this cluster is not used and scipped. VERBOSE.print(arguments, verbose=9) implemented_policies = ['FIFO', 'FILO'] variables = Variables() # docopt for some reason does not show all of the arguments in dot # format that's the reason I used -- format. if arguments.create and \ arguments['--name'] and \ arguments['--cluster'] and \ arguments['--policy']: queue_name = arguments['--name'] cluster_name = arguments['--cluster'] policy = arguments['--policy'] if policy.upper() not in ['FIFO', 'FILO']: Console.error("Policy {policy} not defined, currently " "implemented policies are {policies} ".format( policy=policy.upper(), policies=implemented_policies)) return charge = arguments['--charge'] unit = arguments['--unit'] queue.create(queue_name, cluster_name, policy, charge, unit)
def register(self, file=None): Console.error("Not implemented") raise NotImplementedError
def start(self, security=True): """ start the MongoDB server """ mode = self.data['MODE'] if mode == 'docker': from cloudmesh.mongo.MongoDocker import MongoDocker mongo = MongoDocker() mongo.start(auth=security) # mongo.wait() # mongo.ps() return auth = "" if security: auth = "--auth" mongo_host = self.data['MONGO_HOST'] if platform.lower() == 'win32': try: # command = 'where mongo' # proc = subprocess.Popen(command, shell=True, # stdin=subprocess.PIPE, # stdout=subprocess.PIPE) # out, err = proc.communicate() # print ("MMM", command) # print ("O", out) # print ("E", err) # if out == b'': # Console.error("mongo command not found") # sys.exit() mongo_runner = f"\"{self.mongo_home}\\bin\mongod\" {auth} " \ f"--bind_ip {mongo_host}" \ f" --dbpath \"{self.mongo_path}\" --logpath \"{self.mongo_log}\mongod.log\"" print(mongo_runner) if not os.path.isfile(f'{self.mongo_path}/invisible.vbs'): with open(f'{self.mongo_path}/invisible.vbs', 'w') as f: f.write( 'CreateObject("Wscript.Shell").Run """" & WScript.Arguments(0) & """", 0, False' ) if not os.path.isfile(f'{self.mongo_path}/mongo_starter.bat'): with open(f'{self.mongo_path}/mongo_starter.bat', 'w') as f: f.write(mongo_runner) script = f'wscript.exe \"{self.mongo_path}/invisible.vbs\" \"{self.mongo_path}/mongo_starter.bat\"' print(script) p = subprocess.Popen(script, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) result = "mongod child process should be started successfully." except Exception as e: result = "Mongo in windows could not be started: \n\n" + str(e) else: try: script = f"mongod {auth} --bind_ip {mongo_host}" \ f" --dbpath {self.mongo_path} --logpath {self.mongo_log}/mongod.log --fork" result = Script.run(script) except Exception as e: result = "Mongo could not be started." + str(e) if "successfully" in result: print(Console.ok(result)) else: print(Console.error(result))
def get_run(self, specification): """ function to download file or directory gets the source from the service :param: specification: :return: dict """ source = specification['source'] destination = specification['destination'] recursive = specification['recursive'] trimmed_source = massage_path(source) trimed_dest = massage_path(destination) self.s3_resource, self.s3_client = self.get_s3_resource_client() file_obj = '' try: file_obj = self.s3_client.get_object(Bucket=self.container_name, Key=trimmed_source) except botocore.exceptions.ClientError as e: # object not found Console.error(e) files_downloaded = [] is_target_file = os.path.isfile(trimed_dest) is_target_dir = os.path.isdir(trimed_dest) ''' print('is_target_file') print(is_target_file) print('is_target_dir') print(is_target_dir) ''' if file_obj: try: if is_target_dir: ''' print('target is directory...') print('trimmed_destination : '+ trimmed_destination) print(trimmed_source) ''' os_path_trim_source = os.path.basename(trimmed_source) blob = self.s3_resource.Bucket( self.container_name).download_file( trimmed_source, f"{trimed_dest}/{os_path_trim_source}") else: blob = self.s3_resource.Bucket( self.container_name).download_file( trimmed_source, trimed_dest) # make head call since file download does not return # obj dict to extract meta data metadata = self.s3_client.head_object( Bucket=self.container_name, Key=trimmed_source) files_downloaded.append( extract_file_dict(trimmed_source, metadata)) self.storage_dict['message'] = 'Source downloaded' except FileNotFoundError as e: self.storage_dict['message'] = 'Destination not found' Console.error(e) else: # Search for a directory all_objs = list( self.s3_resource.Bucket( self.container_name).objects.filter(Prefix=trimmed_source)) total_all_objs = len(all_objs) if total_all_objs == 0: self.storage_dict['message'] = 'Source Not Found' elif total_all_objs > 0 and recursive is False: for obj in all_objs: if os.path.basename(obj.key) != self.dir_marker_file_name: if massage_path( obj.key.replace(trimmed_source, '')).count('/') \ == 0: try: blob = self.s3_resource.Bucket( self.container_name ).download_file( obj.key, f"{trimed_dest}/{os.path.basename(obj.key)}" ) # make head call since file download does not # return obj dict to extract meta data metadata = self.s3_client.head_object( Bucket=self.container_name, Key=obj.key) files_downloaded.append( extract_file_dict(obj.key, metadata)) self.storage_dict[ 'message'] = 'Source downloaded' # files_downloaded.append(obj.key) except FileNotFoundError as e: self.storage_dict[ 'message'] = 'Destination not found' Console.error(e) elif total_all_objs > 0 and recursive is True: files_downloaded = [] for obj in all_objs: if os.path.basename(obj.key) != \ self.dir_marker_file_name \ and obj.key[-1] != '/': if massage_path(obj.key.replace(trimmed_source, '')) \ .count('/') == 0: try: blob = self.s3_resource.Bucket( self.container_name ).download_file( obj.key, f"{trimed_dest}/{os.path.basename(obj.key)}" ) # make head call since file download does # not return obj dict to extract meta data metadata = self.s3_client.head_object( Bucket=self.container_name, Key=obj.key) files_downloaded.append( extract_file_dict(obj.key, metadata)) except FileNotFoundError as e: Console.error(e) else: folder_path = massage_path( obj.key.replace(trimmed_source, '').replace( os.path.basename(obj.key), '')) dest_path = f"{trimed_dest}/{folder_path}" try: os.makedirs(dest_path, 0o777) Console.msg() except FileExistsError as e: os.chmod(dest_path, stat.S_IRWXO) Console.error(e) try: blob = self.s3_resource.Bucket( self.container_name ).download_file( # obj.key, trimmedDestination + '/' # + os.path.basename(obj.key)) obj.key, f"{trimed_dest}/folder_path{os.path.basename(obj.key)}" ) # make head call since file download # does not return obj dict to extract meta data metadata = self.s3_client.head_object( Bucket=self.container_name, Key=obj.key) files_downloaded.append( extract_file_dict(obj.key, metadata)) except FileNotFoundError as e: Console.error(e) specification['status'] = 'completed' return specification
def do_openapi(self, args, arguments): """ :: Usage: openapi generate [FUNCTION] --filename=FILENAME [--serverurl=SERVERURL] [--yamlfile=YAML] [--import_class] [--all_functions] [--enable_upload] [--verbose] openapi server start YAML [NAME] [--directory=DIRECTORY] [--port=PORT] [--server=SERVER] [--host=HOST] [--verbose] [--debug] [--fg] [--os] openapi server stop NAME openapi server list [NAME] [--output=OUTPUT] openapi server ps [NAME] [--output=OUTPUT] openapi register add NAME ENDPOINT openapi register filename NAME openapi register delete NAME openapi register list [NAME] [--output=OUTPUT] openapi TODO merge [SERVICES...] [--dir=DIR] [--verbose] openapi TODO doc FILE --format=(txt|md)[--indent=INDENT] openapi TODO doc [SERVICES...] [--dir=DIR] openapi sklearn FUNCTION MODELTAG openapi sklearnreadfile FUNCTION MODELTAG openapi sklearn upload --filename=FILENAME Arguments: FUNCTION The name for the function or class MODELTAG The arbirtary name choosen by the user to store the Sklearn trained model as Pickle object FILENAME Path to python file containing the function or class SERVERURL OpenAPI server URL Default: https://localhost:8080/cloudmesh YAML Path to yaml file that will contain OpenAPI spec. Default: FILENAME with .py replaced by .yaml DIR The directory of the specifications FILE The specification Options: --import_class FUNCTION is a required class name instead of an optional function name --all_functions Generate OpenAPI spec for all functions in FILENAME --debug Use the server in debug mode --verbose Specifies to run in debug mode [default: False] --port=PORT The port for the server [default: 8080] --directory=DIRECTORY The directory in which the server is run --server=SERVER The server [default: flask] --output=OUTPUT The outputformat, table, csv, yaml, json [default: table] --srcdir=SRCDIR The directory of the specifications --destdir=DESTDIR The directory where the generated code is placed Description: This command does some useful things. openapi TODO doc FILE --format=(txt|md|rst) [--indent=INDENT] Sometimes it is useful to generate teh openaopi documentation in another format. We provide fucntionality to generate the documentation from the yaml file in a different formt. openapi TODO doc --format=(txt|md|rst) [SERVICES...] Creates a short documentation from services registered in the registry. openapi TODO merge [SERVICES...] [--dir=DIR] [--verbose] Merges tow service specifications into a single servoce TODO: do we have a prototype of this? openapi sklearn sklearn.linear_model.LogisticRegression Generates the .py file for the Model given for the generator openapi sklearnreadfile sklearn.linear_model.LogisticRegression Generates the .py file for the Model given for the generator which supports reading files openapi generate [FUNCTION] --filename=FILENAME [--serverurl=SERVERURL] [--yamlfile=YAML] [--import_class] [--all_functions] [--enable_upload] [--verbose] Generates an OpenAPI specification for FUNCTION in FILENAME and writes the result to YAML. Use --import_class to import a class with its associated class methods, or use --all_functions to import all functions in FILENAME. These options ignore functions whose names start with '_'. Use --enable_upload to add file upload functionality to a copy of your python file and the resulting yaml file. openapi server start YAML [NAME] [--directory=DIRECTORY] [--port=PORT] [--server=SERVER] [--host=HOST] [--verbose] [--debug] [--fg] [--os] starts an openapi web service using YAML as a specification TODO: directory is hard coded as None, and in server.py it defaults to the directory where the yaml file lives. Can we just remove this argument? openapi server stop NAME stops the openapi service with the given name TODO: where does this command has to be started from openapi server list [NAME] [--output=OUTPUT] Provides a list of all OpenAPI services in the registry openapi server ps [NAME] [--output=OUTPUT] list the running openapi service openapi register add NAME ENDPOINT Openapi comes with a service registry in which we can register openapi services. openapi register filename NAME In case you have a yaml file the openapi service can also be registerd from a yaml file openapi register delete NAME Deletes the names service from the registry openapi register list [NAME] [--output=OUTPUT] Provides a list of all registerd OpenAPI services """ #print(arguments) map_parameters(arguments, 'fg', 'os', 'output', 'verbose', 'port', 'directory', 'yamlfile', 'serverurl', 'name', 'import_class', 'all_functions', 'enable_upload', 'host') arguments.debug = arguments.verbose #VERBOSE(arguments) if arguments.generate: if arguments.import_class and arguments.all_functions: Console.error( 'Cannot generate openapi with both --import_class and --all_functions' ) if arguments.import_class and not arguments.function: Console.error( 'FUNCTION paramter (class name) is required when using --import_class' ) try: p = Parameter(arguments) p.Print() filename = p.filename # ./dir/myfile.py yamlfile = p.yamlfile # ./dir/myfile.yaml directory = p.yamldirectory # ./dir function = p.function # myfunction serverurl = p.serverurl # http://localhost:8080/cloudmesh/ module_name = p.module_name # myfile enable_upload = arguments.enable_upload # append the upload function to the end of a copy of the file if not already done if enable_upload: uploadPython = textwrap.dedent(""" from cloudmesh.openapi.registry.fileoperation import FileOperation def upload() -> str: filename=FileOperation().file_upload() return filename #### upload functionality added """) upload_added = False for line in open(filename): if '#### upload functionality added' in line: upload_added = True if not upload_added: filename_upload = filename.replace( '.py', '_upload-enabled.py') copyfile(filename, filename_upload) Console.info(f'copied {filename} to {filename_upload}') filename = filename_upload module_name = module_name + '_upload-enabled' with open(filename, 'a') as f: f.write('\n') f.write(uploadPython) Console.info( f'added upload functionality to {filename}') # Parameter() takes care of putting the filename in the path imported_module = import_module(module_name) dataclass_list = [] for attr_name in dir(imported_module): attr = getattr(imported_module, attr_name) if is_dataclass(attr): dataclass_list.append(attr) # not currently supporting multiple functions or all functions # could do comma-separated function/class names if enable_upload: upload_obj = getattr(imported_module, 'upload') setattr(sys.modules[module_name], 'upload', upload_obj) if arguments.import_class: class_obj = getattr(imported_module, function) # do we maybe need to do this here? # setattr(sys.modules[module_name], function, class_obj) class_description = class_obj.__doc__.strip().split( "\n")[0] func_objects = {} for attr_name in dir(class_obj): attr = getattr(class_obj, attr_name) if isinstance( attr, types.MethodType) and attr_name[0] != '_': # are we sure this is right? # would probably create a valid openapi yaml, but not technically accurate # module.function may work but it should be module.Class.function setattr(sys.modules[module_name], attr_name, attr) func_objects[attr_name] = attr elif is_dataclass(attr): dataclass_list.append(attr) openAPI = generator.Generator() Console.info('Generating openapi for class: ' + class_obj.__name__) openAPI.generate_openapi_class( class_name=class_obj.__name__, class_description=class_description, filename=filename, func_objects=func_objects, serverurl=serverurl, outdir=directory, yamlfile=yamlfile, dataclass_list=dataclass_list, all_function=False, enable_upload=enable_upload, write=True) elif arguments.all_functions: func_objects = {} for attr_name in dir(imported_module): if type( getattr(imported_module, attr_name) ).__name__ == 'function' and attr_name[0] != '_': func_obj = getattr(imported_module, attr_name) setattr(sys.modules[module_name], attr_name, func_obj) func_objects[attr_name] = func_obj openAPI = generator.Generator() Console.info( 'Generating openapi for all functions in file: ' + filename) openAPI.generate_openapi_class( class_name=module_name, class_description="No description provided", filename=filename, func_objects=func_objects, serverurl=serverurl, outdir=directory, yamlfile=yamlfile, dataclass_list=dataclass_list, all_function=True, enable_upload=enable_upload, write=True) else: func_obj = getattr(imported_module, function) setattr(sys.modules[module_name], function, func_obj) openAPI = generator.Generator() Console.info('Generating openapi for function: ' + func_obj.__name__) openAPI.generate_openapi(f=func_obj, filename=filename, serverurl=serverurl, outdir=directory, yamlfile=yamlfile, dataclass_list=dataclass_list, enable_upload=enable_upload, write=True) except Exception as e: Console.error("Failed to generate openapi yaml") print(e) elif arguments.server and arguments.start and arguments.os: try: s = Server(name=arguments.NAME, spec=path_expand(arguments.YAML), directory=path_expand(arguments.directory) or arguments.directory, port=arguments.port, server=arguments.wsgi, debug=arguments.debug) pid = s.run_os() VERBOSE(arguments, label="Server parameters") print(f"Run PID: {pid}") except FileNotFoundError: Console.error("specification file not found") except Exception as e: print(e) elif arguments.server and arguments.list: try: result = Server.list(name=arguments.NAME) # BUG: order= nt yet defined print(Printer.list(result)) except ConnectionError: Console.error("Server not running") elif arguments.server and arguments.ps: try: print() Console.info("Running Cloudmesh OpenAPI Servers") print() result = Server.ps(name=arguments.NAME) print(Printer.list(result, order=["name", "pid", "spec"])) print() except ConnectionError: Console.error("Server not running") elif arguments.register and arguments.add: registry = Registry() result = registry.add(name=arguments.NAME, url=arguments.BASEURL, pid=arguments.PID) registry.Print(data=result, output=arguments.output) elif arguments.register and arguments.delete: registry = Registry() result = registry.delete(name=arguments.NAME) if result == 0: Console.error("Entry could not be found") else: Console.ok("Ok. Entry deleted") elif arguments.register and arguments.list: registry = Registry() result = registry.list(name=arguments.NAME) registry.Print(data=result, output=arguments.output) elif arguments.register and arguments['filename']: registry = Registry() result = [registry.add_form_file(arguments['filename'])] registry.Print(data=result, output=arguments.output) elif arguments.server and arguments.start: # VERBOSE(arguments) try: s = Server( name=arguments.NAME, spec=path_expand(arguments.YAML), directory=None, #directory=path_expand( # arguments.directory) or arguments.directory, port=arguments.port, host=arguments.host, server=arguments.wsgi, debug=arguments.debug) pid = s.start(name=arguments.NAME, spec=path_expand(arguments.YAML), foreground=arguments.fg) print(f"Run PID: {pid}") except FileNotFoundError: Console.error("specification file not found") except Exception as e: print(e) elif arguments.server and arguments.stop: try: print() Console.info("Stopping Cloudmesh OpenAPI Server") print() Server.stop(name=arguments.NAME) print() except ConnectionError: Console.error("Server not running") elif arguments.sklearn and not arguments.upload: try: Sklearngenerator(input_sklibrary=arguments.FUNCTION, model_tag=arguments.MODELTAG) except Exception as e: print(e) elif arguments.sklearnreadfile and not arguments.upload: try: SklearngeneratorFile(input_sklibrary=arguments.FUNCTION, model_tag=arguments.MODELTAG) except Exception as e: print(e) #TODO: implement this? elif arguments.sklearn and arguments.upload: try: openAPI = generator.Generator() openAPI.fileput() except Exception as e: print(e) '''