def test_001(self): HEADING("assign key=value") v = Variables() n = len(v) v["gregor"] = "gregor" assert (len(v) == n + 1) assert "gregor" in v v.close()
def test_004(self): HEADING("directory and key subtract ") d = {"a": "1", "b": "2"} v = Variables() v + d print(v) assert "a" in v and "b" in v v - d.keys() assert "a" not in v and "b" not in v print(v) v.close()
def postcmd(self, stop, line): StopWatch.stop("command") try: variable = Variables() if "timer" not in variable: variable["timer"] = "off" if variable["timer"].lower() in ['on', 'true']: print("Timer: {:.4f}s ({})".format(StopWatch.get("command"), line.strip())) variable.close() except Exception as e: Error.traceback(error=e) return stop
def get(self, name=None): connection = None if name is None: variables = Variables() cloudname = variables['cloud'] kind = self.config.get( "cloudmesh.cloud.{name}.cm.kind".format(name=name)) credentials = self.config.get( "cloudmesh.cloud.{name}.credentials".format(name=name)) # BUG FROM HERE ON WRONG if kind == 'azure': AZURE_SUBSCRIPTION_ID = credentials['AZURE_SUBSCRIPTION_ID'] AZURE_MANAGEMENT_CERT_PATH = credentials[ 'AZURE_MANAGEMENT_CERT_PATH'] AZDriver = get_driver(Provider.AZURE) connection = AZDriver(subscription_id=AZURE_SUBSCRIPTION_ID, key_file=AZURE_MANAGEMENT_CERT_PATH) elif kind == 'aws': EC2_ACCESS_ID = credentials['EC2_ACCESS_ID'] EC2_SECRET_KEY = credentials['EC2_SECRET_KEY'] EC2Driver = get_driver(Provider.EC2) connection = EC2Driver(EC2_ACCESS_ID, EC2_SECRET_KEY) return connection
def setup(self): banner("setup", c="-") self.user = Config()["cloudmesh"]["profile"]["user"] self.clouduser = '******' self.name_generator = Name(experiment="exp", group="grp", user=self.user, kind="vm", counter=1) self.name = str(self.name_generator) self.name_generator.incr() self.new_name = str(self.name_generator) variables = Variables() # this gives the current default cloud cloud = variables['cloud'] # specify the cloud name to make sure this test # is done for the openstack cloud # self.p = Provider(name="chameleon") self.secgroupname = "CM4TestSecGroup" self.secgrouprule = { "ip_protocol": "tcp", "from_port": 8080, "to_port": 8088, "ip_range": "129.79.0.0/16" } self.testnode = None
def setup(self): banner("setup", c="-") self.user = Config()["cloudmesh"]["profile"]["user"] self.clouduser = '******' self.name_generator = Name(experiment="exp", group="grp", user=self.user, kind="vm", counter=1) self.name = str(self.name_generator) self.name_generator.incr() self.new_name = str(self.name_generator) variables = Variables() cloud = variables['cloud'] self.p = Provider(name=cloud) self.secgroupname = "CM4TestSecGroup" self.secgrouprule = { "ip_protocol": "tcp", "from_port": 8080, "to_port": 8088, "ip_range": "129.79.0.0/16" } self.testnode = None
def test_003(self): HEADING("directory add ") d = {"a": "1", "b": "2"} v = Variables() v + d print(v) assert "a" in v and "b" in v del v["a"] del v["b"] v + d assert "a" in v and "b" in v v - d assert "a" not in v and "b" not in v print(v) v.close()
def __init__(self, config_path='~/.cloudmesh/cloudmesh4.yaml', encrypted=False): """ Initialize the Config class. :param config_path: A local file path to cloudmesh yaml config with a root element `cloudmesh`. Default: `~/.cloudmesh/cloudmesh4.yaml` """ self.__dict__ = self.__shared_state if "data" not in self.__dict__: VERBOSE.print("Load config", verbose=9) self.config_path = Path(path_expand(config_path)).resolve() self.config_folder = dirname(self.config_path) if not exists(self.config_folder): mkdir(self.config_folder) if not isfile(self.config_path): source = Path(join(dirname(realpath(__file__)), "../../etc/cloudmesh4.yaml")) copyfile(source.resolve(), self.config_path) #with open(self.config_path, "r") as stream: # # self.data = yaml.load(stream, Loader=yaml.FullLoader) # self.data = yaml.load(stream, Loader=yaml.SafeLoader) with open(self.config_path, "r") as stream: content = stream.read() content = path_expand(content) content = self.spec_replace(content) self.data = yaml.load(content, Loader=yaml.SafeLoader) # self.data is loaded as nested OrderedDict, can not use set or get methods directly if self.data is None: raise EnvironmentError( "Failed to load configuration file cloudmesh4.yaml, please check the path and file locally") # # populate default variables # self.variable_database = Variables(filename="~/.cloudmesh/var-data") self.set_debug_defaults() default = self.default() for name in self.default(): if name not in self.variable_database: self.variable_database[name] = default[name] if "cloud" in default: self.cloud = default["cloud"] else: self.cloud = None
def test_setup(self): self.variables = Variables() self.storages = Parameter.expand(self.variables['storage']) pytest.storage = self.storages[0] command = ['python', 'server.py'] pytest.openapi = run(command) pytest.openapi.execute() print(pytest.openapi.pid) time.sleep(5)
def do_storage(self, args, arguments): """ :: Usage: storage [--storage=SERVICE] put FILENAME storage [--storage=SERVICE] get FILENAME storage [--storage=SERVICE] delete FILENAME storage [--storage=SERVICE] size FILENAME storage [--storage=SERVICE] info FILENAME storage [--storage=SERVICE] create FILENAME storage [--storage=SERVICE] sync SOURCEDIR DESTDIR This command does some useful things. Arguments: FILE a file name Options: -f specify the file Example: set storage=box starage put FILENAME is the same as starage --storage=box put FILENAME """ pprint(arguments) m = Manager() service = None filename = arguments.FILENAME[0] try: service = arguments["--storage"][0] except Exception as e: try: v = Variables() service = v['storage'] except Exception as e: service = None if service is None: Console.error("storge service not defined") if arguments.get: m.get(service, filename)
def replace_vars(self, line): # self.update_time() variable = Variables() newline = line if len(variable) is not None: for name in variable.data: value = str(variable[name]) newline = newline.replace("$" + name, value) newline = newline.replace("var." + name, value) for v in os.environ: name = v.replace('os.', '') if name in newline: value = os.environ[name] newline = newline.replace("os." + v, value) default = Default() if default is not None: for v in default.data: name = "default." + v.replace(",", ".") value = default.data[v] if name in newline: newline = newline.replace(name, value) # replace if global is missing global_default = default["global"] if global_default is not None: for v in global_default: name = "default." + v value = global_default[v] if name in newline: newline = newline.replace(name, value) default.close() variable.close() newline = path_expand(newline) return line, newline
def do_storage(self, args, arguments): """ :: Usage: storage [--storage=SERVICE] create dir DIRECTORY storage [--storage=SERVICE] get SOURCE DESTINATION [--recursive] storage [--storage=SERVICE] put SOURCE DESTINATION [--recursive] storage [--storage=SERVICE] list SOURCE [--recursive] storage [--storage=SERVICE] delete SOURCE storage [--storage=SERVICE] search DIRECTORY FILENAME [--recursive] This command does some useful things. Arguments: SOURCE SOURCE can be a directory or file DESTINATION DESTINATION can be a directory or file DIRECTORY DIRECTORY refers to a folder on the cloud service Options: --storage=SERVICE specify the cloud service name like aws or azure or box or google Description: commands used to upload, download, list files on different cloud storage services. storage put [options..] Uploads the file specified in the filename to specified cloud from the SOURCEDIR. storage get [options..] Downloads the file specified in the filename from the specified cloud to the DESTDIR. storage delete [options..] Deletes the file specified in the filename from the specified cloud. storage list [options..] lists all the files from the container name specified on the specified cloud. storage create dir [options..] creates a folder with the directory name specified on the specified cloud. storage search [options..] searches for the source in all the folders on the specified cloud. Example: set storage=azureblob storage put SOURCE DESTINATION --recursive is the same as storage --storage=azureblob put SOURCE DESTINATION --recursive """ # arguments.CONTAINER = arguments["--container"] map_parameters(arguments, "recursive", "storage") arguments.storage = arguments["--storage"] pprint(arguments) m = Manager() service = None # # BUG # services = Parameter.expand(arguments.storage) # service = services[0] # if services is None: # ... do second try ##### BUG try: service = arguments["--storage"][0] except Exception as e: try: v = Variables() service = v['storage'] except Exception as e: service = None if service is None: Console.error("storage service not defined") return # bug this is now done twice .... if arguments.storage is None: variables = Variables() arguments.storage = variables['storage'] ##### Prvious code needs to be modified if arguments.get: m.get(arguments.storage, arguments.SOURCE, arguments.DESTINATION, arguments.recursive) elif arguments.put: m.put(arguments.storage, arguments.SOURCE, arguments.DESTINATION, arguments.recursive) elif arguments.list: print('in List') m.list(arguments.storage, arguments.SOURCE, arguments.recursive) elif arguments.create and arguments.dir.: m.createdir(arguments.storage, arguments.DIRECTORY) elif arguments.delete.: m.delete(arguments.storage, arguments.SOURCE) elif arguments['search']: m.search(arguments.storage, arguments.DIRECTORY, arguments.FILENAME, arguments.recursive)
def do_batch(self, args, arguments): """ :: Usage: batch job create --name=NAME --cluster=CLUSTER --script=SCRIPT --executable=EXECUTABLE --destination=DESTINATION --source=SOURCE [--companion-file=COMPANION_FILE] [--outfile-name=OUTPUT_FILE_NAME] [--suffix=SUFFIX] [--overwrite] batch job run [--name=NAMES] [--format=FORMAT] batch job fetch [--name=NAMES] batch job remove [--name=NAMES] batch job clean [--name=NAMES] batch job set [--name=NAMES] PARAMETER=VALUE batch job list [--name=NAMES] [--depth=DEPTH] batch cluster test [--cluster=CLUSTERS] batch cluster list [--cluster=CLUSTERS] [--depth=DEPTH] batch cluster remove [--cluster=CLUSTERS] batch cluster set [--cluster=CLUSTERS] PARAMETER=VALUE Arguments: FILE a file name INPUT_TYPE tbd Options: -f specify the file --depth=DEPTH [default: 1] --format=FORMAT [default: table] Description: This command allows to submit batch jobs to queuing systems hosted in an HBC center as a service directly form your commandline. We assume that a number of experiments are conducted with possibly running the script multiple times. Each experiment will save the batch script in its own folder. The output of the script can be saved in a destination folder. A virtual directory is used to coordinate all saved files. The files can be located due to the use of the virtual directory on multiple different data or file services Authentication to the Batch systems is done viw the underlaying HPC center authentication. We assume that the user has an account to submit on these systems. (SSH, 2 factor, XSEDE-account) TBD. Experiments: experiments are jobs that can be run multiple times and create input and output file sin them cloudmesh: experiment: job: name: {cloudmesh.profile.user.name}-01 directory: ~/experiment/{experiment.job.name} output: {cloudmesh.experiment.job.name}/output input: ~/experiment/{experiment.job.name}/input script: script.sh source ,,, destination: {cloudmesh.experiment.job.directory} I do not know what companion file is Examples: batch job run [--name=NAMES] [--format=FORMAT] runs jobs with the given names LOTS OF DOCUMENTATION MISSING HERE [--companion-file=COMPANION_FILE] [--outfile-name=OUTPUT_FILE_NAME] [--suffix=SUFFIX] [--overwrite] """ # # create slurm manager so it can be used in all commands # slurm_manager = SlurmCluster() # debug=arguments["--debug"]) arguments["--cloud"] = "test" arguments["NAME"] = "fix" map_parameters(arguments, "cloud", "name", "cluster", "script", "type", "destination", "source", "format") # if not arguments.create # find cluster name from Variables() # if no cluster is defined look it up in yaml in batch default: # if not defined there fail # clusters = Parameter.expand(arguments.cluster) # name = Parameters.expand[argumnets.name) # this will return an array of clusters and names of jobs and all cluster # job or clusterc commands will be executed on them # see the vm # # if active: False in the yaml file for the cluster this cluster is not used and scipped. VERBOSE.print(arguments, verbose=9) variables = Variables() # do not use print but use ,Console.msg(), Console.error(), Console.ok() if arguments.tester: print("running ... ") slurm_manager.tester() elif arguments.run and arguments.job: # config = Config()["cloudmesh.batch"] names = Parameter.expand(arguments.name) # clouds, names = Arguments.get_cloud_and_names("refresh", arguments, # variables) data = [] for name in names: entry = SlurmCluster.job_specification() data.append(entry) ''' data = { "cm": { "cloud": "karst_debug", "kind": "batch-job", "name": "job012", }, "batch": { "source": "~/.cloudmesh/batch/dir", "destination": "~/.cloudmesh/dir/", "status": "running" } }''' try: raise NotImplementedError except Exception as e: Console.error("Haha", traceflag=True) pprint(data) print(Printer.flatwrite( data, order=["cm.name", "cm.kind", "batch.status"], header=["Name", "Kind", "Status"], output=arguments.format) ) return "" elif arguments.job and arguments.create: # if not arguments.name: # raise ValueError # assert input_type in ['params', 'params+file'], "Input type can be either params or params+file" # if input_type == 'params+file': # assert arguments.get("--argfile-path") is not None, "Input type is params+file but the input \ # filename is not specified" job_name = arguments.name cluster_name = arguments.cluster script_path = Path(arguments.script) if not script_path.exists(): raise FileNotFoundError executable_path = Path(arguments['--executable']) if not executable_path.exists(): raise FileNotFoundError destination = Path(arguments.destination) if not destination.is_absolute(): Console.error("destination path must be absolute", traceflag=True) raise FileNotFoundError source = Path(arguments.source) if not source.exists(): raise FileNotFoundError if arguments.experiment is None: experiment_name = 'job' + self.suffix_generator() else: experiment_name = arguments.experiment + self.suffix_generator() # overwrite = False if type(arguments.get("--overwrite")) is None else arguments.get("--overwrite") if arguments.get("--companion-file") is None: companion_file = Path() else: companion_file = Path(arguments.get("--companion-file")) slurm_manager.create(job_name, cluster_name, script_path, executable_path, destination, source, experiment_name, companion_file) elif arguments.remove: if arguments.cluster: slurm_manager.remove("cluster", arguments.get("CLUSTER_NAME")) if arguments.job: slurm_manager.remove("job", arguments.get("JOB_NAME")) elif arguments.list: max_depth = 1 if arguments.get("DEPTH") is None else int(arguments.get("DEPTH")) if arguments.get("clusters"): slurm_manager.list("clusters", max_depth) elif arguments.get("jobs"): slurm_manager.list("jobs", max_depth) elif arguments.set: if arguments.get("cluster"): cluster_name = arguments.get("CLUSTER_NAME") parameter = arguments.get("PARAMETER") value = arguments.get("VALUE") slurm_manager.set_param("cluster", cluster_name, parameter, value) if arguments.job: config_name = arguments.get("JOB_NAME") parameter = arguments.get("PARAMETER") value = arguments.get("VALUE") slurm_manager.set_param("job-metadata", config_name, parameter, value) elif arguments.start and arguments.job: job_name = arguments.get("JOB_NAME") slurm_manager.run(job_name) elif arguments.get("fetch"): job_name = arguments.get("JOB_NAME") slurm_manager.fetch(job_name) elif arguments.test: cluster_name = arguments.get("CLUSTER_NAME") slurm_manager.connection_test(cluster_name) elif arguments.clean: job_name = arguments.get("JOB_NAME") slurm_manager.clean_remote(job_name)
def test_002(self): HEADING("delete") v = Variables() del v["gregor"] assert "gregor" not in v v.close()
def do_storage(self, args, arguments): """ :: Usage: storage [--storage=SERVICE] put FILENAME SOURCEDIR storage [--storage=SERVICE] get FILENAME DESTDIR storage [--storage=SERVICE] delete file FILENAME storage [--storage=SERVICE] list file DIRNAME storage [--storage=SERVICE] info FILENAME storage [--storage=SERVICE] create dir DIRNAME storage [--storage=SERVICE] list dir storage [--storage=SERVICE] delete dir DIRNAME This command does some useful things. Arguments: FILENAME a BLOB name SOURCEDIR local path for the FILENAME to be uploaded DESTDIR local path for the FILENAME to be downloaded Options: --storage=SERVICE specify the cloud service name like aws or azure or box or google Description: commands used to upload, download, list files on different cloud storage services. storage put [options..] Uploads the file specified in the filename to specified cloud from the SOURCEDIR. storage get [options..] Downloads the file specified in the filename from the specified cloud to the DESTDIR. storage delete file [options..] Deletes the file specified in the filename from the specified cloud. storage list file [options..] lists all the files from the container name specified on the specified cloud. storage info [options..] returns the properties of the filename specified on the specified cloud. storage create dir [options..] creates a folder with the directory name specified on the specified cloud. storage list dir [options..] lists all the folders on the specified cloud. storage delete dir [options..] deletes all the files in the directory specified on the specified cloud. Example: set storage=azureblob storage put FILENAME SOURCEDIR is the same as storage --storage=azureblob put FILENAME SOURCEDIR """ # arguments.CONTAINER = arguments["--container"] arguments.SERVICE = arguments["--storage"] pprint(arguments) m = Manager() service = None try: service = arguments["--storage"][0] except Exception as e: try: v = Variables() service = v['storage'] except Exception as e: service = None if service is None: Console.error("storage service not defined") if arguments['get']: if arguments.SERVICE is None: variables = Variables() arguments.SERVICE = variables['storage'] m.get(arguments.SERVICE, arguments.FILENAME, arguments.DESTDIR) elif arguments['put']: if arguments.SERVICE is None: variables = Variables() arguments.SERVICE = variables['storage'] m.put(arguments.SERVICE, arguments.FILENAME, arguments.SOURCEDIR) elif arguments['delete'] and arguments['file']: if arguments.SERVICE is None: variables = Variables() arguments.SERVICE = variables['storage'] m.delete(arguments.SERVICE, arguments.FILENAME) elif arguments['list'] and arguments['file']: if arguments.SERVICE is None: variables = Variables() arguments.SERVICE = variables['storage'] m.listfiles(arguments.SERVICE, arguments.DIRNAME) elif arguments['info']: if arguments.SERVICE is None: variables = Variables() arguments.SERVICE = variables['storage'] m.info(arguments.SERVICE, arguments.FILENAME) elif arguments['create'] and arguments['dir']: if arguments.SERVICE is None: variables = Variables() arguments.SERVICE = variables['storage'] m.createdir(arguments.SERVICE, arguments.DIRNAME) elif arguments['list'] and arguments['dir']: if arguments.SERVICE is None: variables = Variables() arguments.SERVICE = variables['storage'] m.listdir(arguments.SERVICE) elif arguments['delete'] and arguments['dir']: if arguments.SERVICE is None: variables = Variables() arguments.SERVICE = variables['storage'] m.deletedir(arguments.SERVICE, arguments.DIRNAME)
def do_image(self, args, arguments): """ :: Usage: image list [NAMES] [--cloud=CLOUD] [--refresh] [--output=OUTPUT] Options: --output=OUTPUT the output format [default: table] --cloud=CLOUD the cloud name --refresh live data taken from the cloud Description: cm image list cm image list --output=csv cm image list 58c9552c-8d93-42c0-9dea-5f48d90a3188 --refresh """ map_parameters(arguments, "refresh", "cloud", "output") VERBOSE.print(arguments, verbose=9) variables = Variables() if arguments.list and arguments.refresh: names = [] clouds, names = Arguments.get_cloud_and_names( "list", arguments, variables) print("AAA", clouds, names) for cloud in clouds: print(f"cloud {cloud}") provider = Provider(name=cloud) images = provider.images() order = provider.p.output['vm']['order'] # not pretty header = provider.p.output['vm']['header'] # not pretty print( Printer.flatwrite(images, sort_keys=["name"], order=order, header=header, output=arguments.output)) return "" elif arguments.list: names = [] clouds, names = Arguments.get_cloud_and_names( "list", arguments, variables) print(clouds, names) try: for cloud in clouds: print(f"List {cloud}") p = Provider(cloud) kind = p.kind collection = "{cloud}-image".format(cloud=cloud, kind=p.kind) db = CmDatabase() vms = db.find(collection=collection) order = p.p.output['vm']['order'] # not pretty header = p.p.output['vm']['header'] # not pretty print( Printer.flatwrite(vms, sort_keys=["name"], order=order, header=header, output=arguments.output)) except Exception as e: VERBOSE.print(e, verbose=9) return ""
def do_vm(self, args, arguments): """ :: Usage: vm ping [NAMES] [--cloud=CLOUDS] [--count=N] [--processors=PROCESSORS] vm check [NAMES] [--cloud=CLOUDS] [--processors=PROCESSORS] vm status [NAMES] [--cloud=CLOUDS] vm console [NAME] [--force] vm start [NAMES] [--cloud=CLOUD] [--dryrun] vm stop [NAMES] [--cloud=CLOUD] [--dryrun] vm terminate [NAMES] [--cloud=CLOUD] [--dryrun] vm delete [NAMES] [--cloud=CLOUD] [--dryrun] vm refresh [--cloud=CLOUDS] vm list [NAMES] [--cloud=CLOUDS] [--output=OUTPUT] [--refresh] vm boot [--name=NAME] [--cloud=CLOUD] [--username=USERNAME] [--image=IMAGE] [--flavor=FLAVOR] [--public] [--secgroup=SECGROUPs] [--key=KEY] [--dryrun] vm boot [--n=COUNT] [--cloud=CLOUD] [--username=USERNAME] [--image=IMAGE] [--flavor=FLAVOR] [--public] [--secgroup=SECGROUPS] [--key=KEY] [--dryrun] vm run [--name=NAMES] [--username=USERNAME] [--dryrun] COMMAND vm script [--name=NAMES] [--username=USERNAME] [--dryrun] SCRIPT vm ip assign [NAMES] [--cloud=CLOUD] vm ip show [NAMES] [--group=GROUP] [--cloud=CLOUD] [--output=OUTPUT] [--refresh] vm ip inventory [NAMES] vm ssh [NAMES] [--username=USER] [--quiet] [--ip=IP] [--key=KEY] [--command=COMMAND] [--modify-knownhosts] vm rename [OLDNAMES] [NEWNAMES] [--force] [--dryrun] vm wait [--cloud=CLOUD] [--interval=SECONDS] vm info [--cloud=CLOUD] [--output=OUTPUT] vm username USERNAME [NAMES] [--cloud=CLOUD] vm resize [NAMES] [--size=SIZE] Arguments: OUTPUT the output format COMMAND positional arguments, the commands you want to execute on the server(e.g. ls -a) separated by ';', you will get a return of executing result instead of login to the server, note that type in -- is suggested before you input the commands NAME server name. By default it is set to the name of last vm from database. NAMES server name. By default it is set to the name of last vm from database. KEYPAIR_NAME Name of the vm keypair to be used to create VM. Note this is not a path to key. NEWNAMES New names of the VM while renaming. OLDNAMES Old names of the VM while renaming. Options: --output=OUTPUT the output format [default: table] -H --modify-knownhosts Do not modify ~/.ssh/known_hosts file when ssh'ing into a machine --username=USERNAME the username to login into the vm. If not specified it will be guessed from the image name and the cloud --ip=IP give the public ip of the server --cloud=CLOUD give a cloud to work on, if not given, selected or default cloud will be used --count=COUNT give the number of servers to start --detail for table, a brief version is used as default, use this flag to print detailed table --flavor=FLAVOR give the name or id of the flavor --group=GROUP give the group name of server --secgroup=SECGROUP security group name for the server --image=IMAGE give the name or id of the image --key=KEY specify a key to use, input a string which is the full path to the private key file --keypair_name=KEYPAIR_NAME Name of the vm keypair to be used to create VM. Note this is not a path to key. --user=USER give the user name of the server that you want to use to login --name=NAME give the name of the virtual machine --force rename/ delete vms without user's confirmation --command=COMMAND specify the commands to be executed Description: commands used to boot, start or delete servers of a cloud vm default [options...] Displays default parameters that are set for vm boot either on the default cloud or the specified cloud. vm boot [options...] Boots servers on a cloud, user may specify flavor, image .etc, otherwise default values will be used, see how to set default values of a cloud: cloud help vm start [options...] Starts a suspended or stopped vm instance. vm stop [options...] Stops a vm instance . vm delete [options...] Delete servers of a cloud, user may delete a server by its name or id, delete servers of a group or servers of a cloud, give prefix and/or range to find servers by their names. Or user may specify more options to narrow the search vm floating_ip_assign [options...] assign a public ip to a VM of a cloud vm ip show [options...] show the ips of VMs vm ssh [options...] login to a server or execute commands on it vm list [options...] same as command "list vm", please refer to it vm status [options...] Retrieves status of last VM booted on cloud and displays it. vm refresh [--cloud=CLOUDS] this command refreshes the data for virtual machines, images and flavors for the specified clouds. vm ping [NAMES] [--cloud=CLOUDS] [--count=N] [--processors=PROCESSORS] pings the specified virtual machines, while using at most N pings. The ping is executed in parallel. If names are specifies the ping is restricted to the given names in parameter format. If clouds are specified, names that are not in these clouds are ignored. If the name is set in the variables this name is used. Tip: give the VM name, but in a hostlist style, which is very convenient when you need a range of VMs e.g. sample[1-3] => ['sample1', 'sample2', 'sample3'] sample[1-3,18] => ['sample1', 'sample2', 'sample3', 'sample18'] Quoting commands: cm vm login gvonlasz-004 --command=\"uname -a\" Limitations: Azure: rename is not supported """ map_parameters(arguments, 'active', 'cloud', 'command', 'dryrun', 'flavor', 'force', 'output', 'group', 'image', 'interval', 'ip', 'key', 'modify-knownhosts', 'n', 'name', 'public', 'quiet', 'secgroup', 'size', 'username') VERBOSE.print(arguments, verbose=9) variables = Variables() if arguments.refresh: names = [] clouds, names = Arguments.get_cloud_and_names( "refresh", arguments, variables) return "" elif arguments.ping: # TODO: IMPLEMENT names = [] pings = int(arguments.N or 3) names = [] clouds, names = Arguments.get_cloud_and_names( "ping", arguments, variables) for name in names: ping = Shell.live("ping -c {N} {name}".format(name=name, N=arguments.N)) print(ping) else: return True return "" elif arguments.check: names = [] clouds, names = Arguments.get_cloud_and_names( "check", arguments, variables) return "" elif arguments.status: names = [] clouds, names = Arguments.get_cloud_and_names( "status", arguments, variables) return "" elif arguments.start: names = [] clouds, names = Arguments.get_cloud_and_names( "start", arguments, variables) return "" elif arguments.stop: names = [] clouds, names = Arguments.get_cloud_and_names( "stop", arguments, variables) return "" elif arguments.terminate: names = [] clouds, names = Arguments.get_cloud_and_names( "terminate", arguments, variables) return "" elif arguments.delete: clouds, names = Arguments.get_cloud_and_names( "delete", arguments, variables) return "" elif arguments.boot: print("boot the vm") elif arguments.list: # vm list [NAMES] # [--cloud=CLOUDS] # [--output=OUPTUT] # [--refresh] # if no clouds find the clouds of all specified vms by name # find all vms of the clouds, # print only those vms specified by name, if no name is given print all for the cloud # print("list the vms") clouds, names = Arguments.get_cloud_and_names( "list", arguments, variables) # print("Clouds:", clouds) if arguments.NAMES is not None: names = Parameter.expand(arguments.NAMES) Console.error("NAMES, not yet implemented" + str(names)) try: if arguments["--refresh"]: pass # find all clouds in db # iterate over the clouds # for each name in name queue, find it and add it to the cloud vm list # for each cloud print the vms else: pass # find all clouds in db # iterate over all clouds # find the vm with the name # add it to the cloud list # for each cloud print the vms except Exception as e: VERBOSE.print(e, verbose=9) return "" else: try: if arguments["--refresh"]: for cloud in clouds: Console.ok("refresh " + cloud) p = Provider(cloud) vms = p.list() order = p.p.output['vm']['order'] # not pretty header = p.p.output['vm']['header'] # not pretty print( Printer.flatwrite(vms, sort_keys=["cm.name"], order=order, header=header, output=arguments.output)) else: for cloud in clouds: p = Provider(cloud) kind = p.kind # pprint(p.__dict__) # pprint(p.p.__dict__) # not pretty collection = "{cloud}-node".format(cloud=cloud, kind=p.kind) db = CmDatabase() vms = db.find(collection=collection) # pprint(vms) # print(arguments.output) # print(p.p.output['vm']) order = p.p.output['vm']['order'] # not pretty header = p.p.output['vm']['header'] # not pretty print( Printer.flatwrite(vms, sort_keys=["cm.name"], order=order, header=header, output=arguments.output)) except Exception as e: VERBOSE.print(e, verbose=9) return "" elif arguments.info: """ vm info [--cloud=CLOUD] [--output=OUTPUT] """ print("info for the vm") cloud, names = Arguments.get_cloud_and_names( "info", arguments, variables) elif arguments.rename: print("rename the vm") v = Variables() cloud = v["cloud"] p = Provider(cloud) try: oldnames = Parameter.expand(arguments["OLDNAMES"]) newnames = Parameter.expand(arguments["NEWNAMES"]) force = arguments["--force"] if oldnames is None or newnames is None: Console.error("Wrong VMs specified for rename", traceflag=False) elif len(oldnames) != len(newnames): Console.error("The number of VMs to be renamed is wrong", traceflag=False) else: print(oldnames) print(newnames) for i in range(0, len(oldnames)): oldname = oldnames[i] newname = newnames[i] if arguments["--dryrun"]: Console.ok("Rename {} to {}".format( oldname, newname)) else: print(f"rename {oldname} -> {newname}") p.rename(source=oldname, destination=newname) msg = "info. OK." Console.ok(msg) except Exception as e: Error.traceback(e) Console.error("Problem renameing instances", traceflag=True) elif arguments["ip"] and arguments["show"]: print("show the ips") """ vm ip show [NAMES] [--group=GROUP] [--cloud=CLOUD] [--output=OUTPUT] [--refresh] """ elif arguments["ip"] and arguments["assign"]: """ vm ip assign [NAMES] [--cloud=CLOUD] """ print("assign the public ip") elif arguments["ip"] and arguments["inventory"]: """ vm ip inventory [NAMES] """ print("list ips that could be assigned") elif arguments.username: """ vm username USERNAME [NAMES] [--cloud=CLOUD] """ print("sets the username for the vm") elif arguments.default: print("sets defaults for the vm") elif arguments.run: """ vm run [--name=NAMES] [--username=USERNAME] [--dryrun] COMMAND """ pass elif arguments.script: """ vm script [--name=NAMES] [--username=USERNAME] [--dryrun] SCRIPT """ pass elif arguments.resize: """ vm resize [NAMES] [--size=SIZE] """ pass elif arguments.ssh: """ vm ssh [NAMES] [--username=USER] [--quiet] [--ip=IP] [--key=KEY] [--command=COMMAND] [--modify-knownhosts] """ print("ssh the vm") elif arguments.console: # vm console [NAME] [--force] names = Arguments.get_names(arguments, variables) for name in names: # r = vm.console(name,force=argument.force) Console.msg("{label} {name}".format(label="console", name=name)) return elif arguments.wait: """ vm wait [--cloud=CLOUD] [--interval=SECONDS] """ print("waits for the vm till its ready and one can login")
def do_storage(self, args, arguments): """ :: Usage: storage [--storage=SERVICE] create dir DIRECTORY storage [--storage=SERVICE] get SOURCE DESTINATION [--recursive] storage [--storage=SERVICE] put SOURCE DESTINATION [--recursive] storage [--storage=SERVICE] list SOURCE [--recursive] [--output=OUTPUT] storage [--storage=SERVICE] delete SOURCE storage [--storage=SERVICE] search DIRECTORY FILENAME [--recursive] [--output=OUTPUT] storage [--storage=SERVICE] sync SOURCE DESTINATION [--name=NAME] [--async] storage [--storage=SERVICE] sync status [--name=NAME] storage config list [--output=OUTPUT] This command does some useful things. Arguments: SOURCE SOURCE can be a directory or file DESTINATION DESTINATION can be a directory or file DIRECTORY DIRECTORY refers to a folder on the cloud service Options: --storage=SERVICE specify the cloud service name like aws or azure or box or google Description: commands used to upload, download, list files on different cloud storage services. storage put [options..] Uploads the file specified in the filename to specified cloud from the SOURCEDIR. storage get [options..] Downloads the file specified in the filename from the specified cloud to the DESTDIR. storage delete [options..] Deletes the file specified in the filename from the specified cloud. storage list [options..] lists all the files from the container name specified on the specified cloud. storage create dir [options..] creates a folder with the directory name specified on the specified cloud. storage search [options..] searches for the source in all the folders on the specified cloud. sync SOURCE DESTINATION puts the content of source to the destination. If --recursive is specified this is done recursively from the source If --async is specified, this is done asyncronously If a name is specified, the process can also be monitored with the status command by name. If the anme is not specified all date is monitored. sync status The status for the asynchronous sync can be seen with this command config list Lists the configures storage services in the yaml file Example: set storage=azureblob storage put SOURCE DESTINATION --recursive is the same as storage --storage=azureblob put SOURCE DESTINATION --recursive """ # arguments.CONTAINER = arguments["--container"] map_parameters(arguments, "recursive", "storage") VERBOSE(arguments) if arguments.storage is None: try: v = Variables() arguments.storage = v['storage'] except Exception as e: arguments.storage = None raise ValueError("Storage provider is not defined") arguments.storage = Parameter.expand(arguments.storage) # # BUG: some commands could be run on more than the first provider, # such as list # thus the if condition needs to be reorganized if arguments["get"]: provider = Provider(arguments.storage[0]) result = provider.get(arguments.storage, arguments.SOURCE, arguments.DESTINATION, arguments.recursive) elif arguments.put: provider = Provider(arguments.storage[0]) result = provider.put(arguments.storage, arguments.SOURCE, arguments.DESTINATION, arguments.recursive) elif arguments.create and arguments.dir: provider = Provider(arguments.storage[0]) result = provider.createdir(arguments.storage, arguments.DIRECTORY) elif arguments.list: # # BUG: this command is much more complicated # for storage in arguments.storage: provider = Provider(storage) result = provider.list(arguments.storage, arguments.SOURCE, arguments.recursive) elif arguments.delete: # # BUG:: this command could be much more complicated # for storage in arguments.storage: provider = Provider(storage) provider.delete(arguments.storage, arguments.SOURCE) elif arguments.search: # # BUG: this command is much more complicated # for storage in arguments.storage: provider = Provider(storage) provider.search(arguments.storage, arguments.DIRECTORY, arguments.FILENAME, arguments.recursive) elif arguments.rsync: # TODO: implement raise NotImplementedError
def onecmd(self, line): """Interpret the argument as though it had been typed in response to the prompt. This may be overridden, but should not normally need to be; see the precmd() and postcmd() methods for useful execution hooks. The return value is a flag indicating whether interpretation of commands by the interpreter should stop. """ oldline, line = self.replace_vars(line) # ----------------------------- # print comment lines, but do not execute them # ----------------------------- if line.startswith('#') \ or line.startswith('//') \ or line.startswith('/*'): print(line) return "" if line.startswith('!'): os.system(line[1:]) return "" # if line is None: # return "" # if line.startswith("!"): # line.replace("!", "! ") # line = self.var_replacer(line) # if line != "hist" and line: # self._hist += [line.strip()] # if line.startswith("!") or line.startswith("shell"): # self.do_shell_exec(line[1:]) # return "" cmd, arg, line = self.parseline(line) if line.startswith("$") or line.startswith('var.'): line = line.replace("$", "", 1) line = line.replace("var.", "", 1) print("FIND>", line, "<", sep='') variable = Variables() print(variable[line]) variable.close() return "" # ----------------------------- # handle empty line # ----------------------------- if not line: return self.emptyline() # ----------------------------- # handle file execution # ----------------------------- # # this does not yet work # # if os.path.isfile(line): # print ("... execute", line) # self.do_exec(line) # return "" if cmd != '': try: func = getattr(self, 'do_' + cmd) return func(arg) except AttributeError as e: print("ERROR: command error while executing '", cmd, "'", sep='') cmd = None line = oldline # Error.traceback(error=e, debug=True, trace=True) return ""
def print(msg, verbose=0, label=None, color="BLUE"): _verbose = int(Variables()["verbose"] or 0) if _verbose >= verbose: banner(pformat(msg), label=label, color=color)
def do_key(self, args, arguments): """ :: Usage: key -h | --help key list --cloud=CLOUDS [--format=FORMAT] key list --source=ssh [--dir=DIR] [--format=FORMAT] key list --source=git [--format=FORMAT] [--username=USERNAME] key list [NAMES] [--format=FORMAT] key load --filename=FILENAME [--format=FORMAT] key add [NAME] [--source=FILENAME] key add [NAME] [--source=git] key add [NAME] [--source=ssh] key get NAME [--format=FORMAT] key default --select key delete (NAMES | --select | --all) [--dryrun] key delete NAMES --cloud=CLOUDS [--dryrun] key upload [NAMES] [--cloud=CLOUDS] [--dryrun] key upload [NAMES] [VMS] [--dryrun] key group upload [--group=GROUPNAMES] [--cloud=CLOUDS] [--dryrun] key group add [--group=GROUPNAMES] [--cloud=CLOUDS] [--dryrun] key group add --file=FILENAME key group delete [--group=GROUPNAMES] [NAMES] [--dryrun] key group list [--group=GROUPNAMES] [--format=FORMAT] key group export --group=GROUNAMES --filename=FILENAME Arguments: VMS Para,eterized list of virtual machines CLOUDS The clouds NAME The name of the key. SOURCE db, ssh, all KEYNAME The name of a key. For key upload it defaults to the default key name. FORMAT The format of the output (table, json, yaml) FILENAME The filename with full path in which the key is located Options: --dir=DIR the directory with keys [default: ~/.ssh] --format=FORMAT the format of the output [default: table] --source=SOURCE the source for the keys [default: cm] --username=USERNAME the source for the keys [default: none] --name=KEYNAME The name of a key Description: Please note that some values are read from the cloudmesh4.yaml file. One such value is cloudmesh.profile.user Manages public keys is an essential component of accessing virtual machine sin the cloud. There are a number of sources where you can find public keys. This includes teh ~/.ssh directory and for example github. To list these keys the following list functions are provided. key list --source=git [--username=USERNAME] lists all keys in git for the specified user. If the name is not specified it is read from cloudmesh4.yaml key list --source=ssh [--dir=DIR] [--format=FORMAT] lists all keys in the directory. If the directory is not specified the default will be ~/.ssh key list NAMES lists all keys in the named virtual machines. The keys will be uploaded into cloudmesh with the add command under the given name. If the name is not specified the name cloudmesh.profile.user is assumed. key add --ssh adds the default key in ~/.ssh/id_rsa.pub key add NAME --source=FILENAME adds the key specifid by the filename with the given name key add NAME --git --username=username adds a named github key from a user with the given github username. Once the keys are uploaded to github, they can be listed key list [NAME] [--format=FORMAT] list the keys loaded to cloudmesh in the giiven format: json, yaml, table. table is default. The NAME cabn be specified and if ommitted the name cloudmesh.profile.user is assumed. key get NAME Retrieves the key indicated by the NAME parameter from cloudmesh and prints its details. key default --select Select the default key interactively key delete NAMES deletes the keys. This may also have an impact on groups key rename NAME NEW renames the key from NAME to NEW. Group management of keys is an important concept in cloudmesh, allowing multiple users to be added to virtual machines. The keys must be uploaded to cloudmesh with a name so they can be used in a group. The --dryrun option executes the command without uploading the information to the clouds. If no groupname is specified the groupname default is assumed. If no cloudnames are specified, all active clouds are assumed. active clouds can be set in the cloudmesh4.yaml file. key group delete [GROUPNAMES] [NAMES] [--dryrun] deletes the named keys from the named groups. key group list [GROUPNAMES] [--format=FORMAT] list the key names and details in the group. key group upload [GROUPNAMES] [CLOUDS] [--dryrun] uploads the named groups to the specified clouds. In some cases you may want to store the public keys in files. For this reason we support the following commands. key group add --group=GROUPNAME --file=FILENAME the command adds the keys to the given group. The keys are written in the files in yaml format. key group export --group=GROUNAMES --filename=FILENAME the command exports the keys to the given group. The keys are written in the files in yaml format. The yaml format is as follows: cloudmesh: keys: NAMEOFKEY: name: NAMEOFKEY key: ssh-rsa AAAA..... comment group: - GROUPNAME ... If a key is included in multiple groups they will be added to the grouplist of the key """ arguments.cloud = arguments['--cloud'] arguments.format = arguments['--format'] arguments.source = arguments['--source'] arguments.dir = arguments['--dir'] pprint(arguments) invalid_names = ['tbd', 'none', "", 'id_rsa'] m = Manager() if arguments.list and arguments.source == "git": # this is much simpler config = Config() username = config["cloudmesh.profile.github"] print("Username:"******"name"], order=["id", "name", "fingerprint", "source"], header=["Id", "Name", "Fingerprint", "Source"])) return "" elif arguments.list and arguments.source == "ssh": # this is much simpler sshkey = SSHkey() print( Printer.flatwrite( [sshkey], sort_keys=["name"], order=["name", "type", "fingerprint", "comment"], header=["Name", "Type", "Fingerprint", "Comment"])) return "" elif arguments.list and arguments.cloud: clouds = Parameter.expand(arguments.cloud) print(clouds) if len(clouds) == 0: variables = Variables() cloudname = variables['cloud'] clouds = [cloudname] cloudkey = [] for cloud in clouds: print(cloud) provider = Provider(clouds) cloudkey.append(provider.keys()) print( Printer.flatwrite( [cloudkey], sort_keys=["name"], order=["name", "type", "fingerprint", "comment"], header=["Name", "Type", "Fingerprint", "Comment"])) return "" elif arguments.list and arguments.source == "db": if arguments.NAMES: names = Parameter.expand(arguments.NAMES) print("find the keys of the following vms", names) print("the keys will be read from mongo") return "" return ""
def do_var(self, args, arguments): """ Usage: var list var clear var delete NAME var NAME=VALUE var NAME Arguments: NAME the name of the variable VALUE the value of the variable FILENAME the filename of the variable Description: Manage persistent variables var NAME=VALUE sets the variable with the name to the value if the value is one of data, time, now it will be replaced with the value at this time, the format will be date 2017-04-14 time 11:30:33 now 2017-04-14 11:30:41 It will wbe replaced accordingly The value can also refer to another variable name. In this case the current value will be copied in the named variable. As we use the $ sign it is important to distinguish shell variables from cms variables while using proper quoting. Examples include: cms var a=\$b cms var 'a=$b' cms var a=val.b The previous command copy the value from b to a. The val command was added to avoid quoting. """ # print (arguments) database = Variables(filename="~/.cloudmesh/var-data") if arguments["clear"]: database.clear() elif arguments["list"]: for name in database: value = database[name] print(name, "=", "'", value, "'", sep="") elif arguments.delete: del database[arguments.NAME] elif arguments.NAME is not None: print(database[arguments.NAME]) elif arguments["NAME=VALUE"] is not None: name, value = arguments["NAME=VALUE"].split("=", 1) if value == "time": value = datetime.now().strftime("%H:%M:%S") elif value == "date": value = datetime.now().strftime("%Y-%m-%d") elif value == "now": value = datetime.now().strftime("%Y-%m-%d %H:%M:%S") elif value.startswith("value."): var = value.replace("value.", "") value = database[var] elif value.startswith("$"): var = value.replace("$", "") value = database[var] print(name, "=", "'", value, "'", sep="") database[name] = value
def do_queue(self, args, arguments): """ :: Usage: queue create --name=NAME --policy=POLICY --cluster=CLUSTER [--charge=CHARGE] [--unit=UNIT] queue activate [--name=NAME] queue deactivate [--name=NAME] queue set unit queue connection_test --job=JOB queue cluster list [--cluster=CLUSTERS] [--depth=DEPTH] queue cluster remove [--cluster=CLUSTERS] queue cluster set [--cluster=CLUSTERS] PARAMETER=VALUE Arguments: FILE a file name INPUT_TYPE tbd Options: -f specify the file --depth=DEPTH [default: 1] --format=FORMAT [default: table] Description: This command creates a queue that is associated with a cloud. We assume that a number of experiments are conducted with possibly running the script multiple times. Each experiment will save the batch script in its own folder. The output of the script can be saved in a destination folder. A virtual directory is used to coordinate all saved files. The files can be located due to the use of the virtual directory on multiple different data or file services Authentication to the Batch systems is done viw the underlaying HPC center authentication. We assume that the user has an account to submit on these systems. (SSH, 2 factor, XSEDE-account) TBD. Examples: LOTS OF DOCUMENTATION MISSING HERE [--companion-file=COMPANION_FILE] [--outfile-name=OUTPUT_FILE_NAME] [--suffix=SUFFIX] [--overwrite] """ # # create slurm manager so it can be used in all commands # queue = Queue() # debug=arguments["--debug"]) # arguments["--cloud"] = "test" # arguments["NAME"] = "fix" # map_parameters(arguments, # "cloud", # "name", # "cluster", # "script", # "type", # "destination", # "source", # "format") # if not arguments.create # find cluster name from Variables() # if no cluster is defined look it up in yaml in batch default: # if not defined there fail # clusters = Parameter.expand(arguments.cluster) # name = Parameters.expand[argumnets.name) # this will return an array of clusters and names of jobs and all cluster # job or clusterc commands will be executed on them # see the vm # # if active: False in the yaml file for the cluster this cluster is not used and scipped. VERBOSE.print(arguments, verbose=9) implemented_policies = ['FIFO', 'FILO'] variables = Variables() # docopt for some reason does not show all of the arguments in dot # format that's the reason I used -- format. if arguments.create and \ arguments['--name'] and \ arguments['--cluster'] and \ arguments['--policy']: queue_name = arguments['--name'] cluster_name = arguments['--cluster'] policy = arguments['--policy'] if policy.upper() not in ['FIFO', 'FILO']: Console.error("Policy {policy} not defined, currently " "implemented policies are {policies} ".format( policy=policy.upper(), policies=implemented_policies)) return charge = arguments['--charge'] unit = arguments['--unit'] queue.create(queue_name, cluster_name, policy, charge, unit)