class SecgroupDatabase: # noinspection PyShadowingBuiltins def __init__(self, kind=None): self.kind = kind self.cm = CmDatabase() self.cloud = "local" self.collection = f"{self.cloud}-{kind}" def clear(self): self.cm.collection(self.collection).delete_many({}) def find(self, name=None): if name is None: query = {} else: query = {'cm.name': name} entries = self.cm.find(collection=self.collection, query=query) return entries def remove(self, name=None): if name is None: query = {} else: query = {'cm.name': name} entries = self.cm.delete(collection=self.collection, **query) return entries # noinspection PyBroadException def list(self, name=None): found = [] if name is None: # find all groups in the db found = self.find() else: # find only the groups specified in the db groups = Parameter.expand(name) # noinspection PyUnusedLocal for group in groups: # noinspection PyUnusedLocal try: entry = self.find(name=name)[0] found.append(entry) except Exception as e: pass return found def update_dict_list(self, entries): for entry in entries: entry['cm'] = { "kind": self.kind, "name": entry['name'], "cloud": self.cloud } return entries
def get_google_pricing(refresh=False): # connect to cm db and check for Google info cm = CmDatabase() googleinfo = cm.collection('gcp-frugal') if googleinfo.estimated_document_count() > 0 and not refresh: Console.msg(f"Using local db gcp flavors...") return googleinfo else: Console.msg(f"Pulling gcp flavor price information...") googleinfo = requests.get( 'https://cloudpricingcalculator.appspot.com/static/data/pricelist.json?v=1570117883807' ).json()['gcp_price_list'] google_list = [] for machine, locations in googleinfo.items(): if type( locations ) is dict and 'cores' in locations and 'memory' in locations: cores = locations['cores'] if cores == 'shared': continue memory = locations['memory'] for location in locations: # 'cores' is end of regions, so stop if found if location == 'cores': break else: if type(locations[location]) is str: print(locations[location]) google_list.append( np.array([ 'gcp', machine, location, float(cores), float(memory), float(locations[location]) ])) googleinforeturn = np.stack(google_list, axis=0) googleinfo = np.stack(googleinforeturn, axis=0) googleinfo = helpers.format_mat(googleinfo) # convert to list of dicts googleinfo = googleinfo.to_dict('records') # write back to cm db for entry in googleinfo: entry["cm"] = { "kind": 'frugal', "driver": 'gcp', "cloud": 'gcp', "name": str(entry['machine-name'] + '-' + entry['location']), "updated": str(datetime.utcnow()), } Console.msg(f"Writing back to db ...") cm.update(googleinfo, progress=True) return cm.collection('gcp-frugal')
def test_list_refresh(self): HEADING() Benchmark.Start() result = Shell.execute("cms frugal list --refresh", shell=True) Benchmark.Stop() cm = CmDatabase() # shoudl we have frugal beeing first??? gregor is not sure # is is asw-frugal or frugal-aws based on local-key it may be # frugal second just as you have. #^ Collection names are created using the cm dictionary assert cm.collection('aws-frugal').estimated_document_count() > 0 assert cm.collection('gcp-frugal').estimated_document_count() > 0 assert cm.collection('azure-frugal').estimated_document_count() > 0 VERBOSE('frugal list refresh complete') VERBOSE(result)
def delete(self, name=None): """ :param name: :return: """ cm = CmDatabase() collection = cm.collection(self.collection) if name is None: query = {} else: query = {'name': name} entries = cm.delete(collection=self.collection, **query) return entries
def delete(self, name=None): """ delete item from registry :param name: name of the item in registry :return: """ cm = CmDatabase() collection = cm.collection(self.collection) if name is None: query = {} else: query = {'name': name} entries = cm.delete(collection=self.collection, **query) return entries
def create_azure_vm(): print("intializing...") Shell.execute("cms init", shell=True) print("setting cloud...") Shell.execute("cms set cloud=azure", shell=True) print("adding key...") Shell.execute("cms key add austin --source=ssh", shell=True) print("getting database information...") cm = CmDatabase() if cm.collection('austin-vm-azure_a1').estimated_document_count() > 0: print("specified vm already exists... deleting old vm...") azure_delete() azure_boot() else: print("specified vm does not exist... booting vm...") azure_boot()
def list(self, name=None): cm = CmDatabase() result = [] if name: col = cm.collection(name=f"{self.cloud}-{self.name}") entries = col.find_one({"cm.kind": 'group', "cm.cloud": 'local', "cm.name": name }, {"_id": 0}) return [entries] else: entries = cm.find(collection=f"{self.cloud}-{self.name}") for entry in entries: result.append(entry) return result
def get_azure_pricing(refresh=False): cm = CmDatabase() azureinfo = cm.collection('azure-frugal') # check to see if azure-frugal already exists in db if azureinfo.estimated_document_count() > 0 and not refresh: Console.msg(f"Using local db azure flavors...") return azureinfo # get local db azure flavors (empty if it does not exist yet) azureinfo = cm.collection('azure-flavor') # create provider to get region try: azureprovider = azureprv.Provider( name='azure', configuration="~/.cloudmesh/cloudmesh.yaml") except: Console.msg("No azure credentials") return region = azureprovider.LOCATION priceregion = location_conv_dict[region] if azureinfo.estimated_document_count() == 0 or refresh: # use provider to fetch info Console.msg(f"Pulling azure flavors...") azureinfo = azureprovider.flavors() else: # use local, just turn it into a list for matching iteration use azureinfo = list(azureinfo.find()) # get pricing and limit to what is needed Console.msg(f"Pulling azure flavor price information...") pricing = requests.get( 'https://azure.microsoft.com/api/v3/pricing/virtual-machines/calculator/?culture=en-us&discount=mosp&v=20191002-1500-96990').json() offers = pricing['offers'] modoffers = {} for key, val in offers.items(): newkey = key.replace('_', '').replace('-', '') modoffers[newkey] = val azure_list = [] for flavor in azureinfo: key = flavor['name'].lower() if key[0] is 'b': # basic search = 'linux' + key[6:].replace('_', '').replace('-', '').replace( 'promo', '') + 'basic' elif key[0] is 's': # standard search = 'linux' + key[9:].replace('_', '').replace('-', '').replace( 'promo', '') + 'standard' elif key[0] is 'l': # low_priority search = 'linux' + key[13:].replace('_', '').replace('-', '').replace( 'promo', '') + 'standard' else: print('no matches on key') continue try: found = modoffers[search] except: # print('machine match failure on ' + search) continue # now to fit it into the frame azure_list.append(np.array( ['azure', flavor['name'], region, found['cores'], found['ram'], found['prices']['perhour'][priceregion]['value']])) azureinfo = np.stack(azure_list, axis=0) azureinfo = helpers.format_mat(azureinfo) # convert to list of dicts azureinfo = azureinfo.to_dict('records') # write back to cm db for entry in azureinfo: entry["cm"] = { "kind": 'frugal', "driver": 'azure', "cloud": 'azure', "name": str(entry['machine-name'] + '-' + entry['location']), "updated": str(datetime.utcnow()), } Console.msg(f"Writing back to db ...") cm.update(azureinfo) return cm.collection('azure-frugal')
class WorkFlowDB(object): def __init__(self, name="workflow", active=False): self.database = CmDatabase() self.workflow_name = name self.collection = self.database.collection(f"{name}-flow") if active: self.switch_to_active_flow() def attributes(self, name): data = { "cm": { "kind": "flow", "cloud": self.workflow_name, "name": name }, "kind": "flow", "cloud": self.workflow_name, "name": name, "status": "defined" } return data @DatabaseUpdate() def add_node(self, node): name = node["name"] node.update(self.attributes(name)) VERBOSE(node) return node def add_edge(self, node, depends_on): self.collection.update_one({"name": node}, {"$push": { "dependencies": depends_on }}) def _node_from_db(self, db_obj): reconstructed = Node(db_obj["name"]) reconstructed.workflow = self.workflow_name reconstructed.dependencies = db_obj["dependencies"] reconstructed.status = db_obj.get("status", "pending") reconstructed.result = db_obj.get("result", {}) reconstructed.progress = "" reconstructed.modified = "" reconstructed.done = "" return reconstructed def remove_node(self, name): self.collection.delete_one({"name": name}) self.collection.update_many({}, {"$pull": {"dependencies": "name"}}) def remove_edge(self, node, depends_on): self.collection.update_one({"name": node}, {"$pull": { "dependencies": depends_on }}) def get_node(self, name=None): return self._node_from_db(self.collection.find_one({"name": name})) def list(self, node=None, edge=None): query = {} if node: query["name"] = node if edge: query["dependencies"] = edge return self.collection.find(query) def list_nodes(self): return [self._node_from_db(node) for node in self.list()] def list_edges(self): return self.collection.aggregate([{ "$unwind": "$dependencies" }, { "$project": { "to": "$name", "from": "$dependencies" } }]) def list_all_workflows(self): all_colls = self.database.collections() return [ name for name in all_colls if "flow" in name and "active" not in name ] def set_node_status(self, node, status): return self.collection.update_one({"name": node}, {"$set": { "status": status }}) def find_root_nodes(self): root_nodes = self.collection.find({ "dependencies.0": { "$exists": False }, "status": "pending" }) return [self._node_from_db(node) for node in root_nodes] def switch_to_active_flow(self): started_collection = f"{self.workflow_name}-flow-active" self.collection = self.database.collection(started_collection) def resolve_node_dependencies(self, name=None): return self.collection.update_many({"dependencies": name}, {"$pull": { "dependencies": name }}) def add_specification(self, spec): pass def start_flow(self): started_collection = f"{self.workflow_name}-flow-active" self.collection.aggregate([{ "$project": { "dependencies": 1, "cm": 1, "kind": 1, "cloud": 1, "name": 1, "status": "pending" } }, { "$out": started_collection }]) self.switch_to_active_flow() def add_node_result(self, nodename, result): return self.collection.update_one({"name": nodename}, {"$set": { "result": result }}) def add_graph(self, yamlfile): pass def last_update(self, workflow=None): """ This method returns the last modified value associated with a database update to a node. :param workflow: The name of the workflow :type workflow: string :return: The time of the last update :rtype: string """ raise NotImplementedError t = "the time string" return t
def do_vm(self, args, arguments): """ :: Usage: vm ping [NAMES] [--cloud=CLOUDS] [--count=N] vm check [NAMES] [--cloud=CLOUDS] [--username=USERNAME] vm status [NAMES] [--cloud=CLOUDS] [--output=OUTPUT] vm console [NAME] [--force] vm log [NAME] [--force] vm stop [NAMES] [--dryrun] vm start [NAMES] [--dryrun] vm terminate [NAMES] [--cloud=CLOUD] [--dryrun] vm delete [NAMES] [--cloud=CLOUD] [--dryrun] vm refresh [--cloud=CLOUDS] vm list [NAMES] [--cloud=CLOUDS] [--output=OUTPUT] [--refresh] vm boot [--n=COUNT] [--name=VMNAMES] [--cloud=CLOUD] [--username=USERNAME] [--image=IMAGE] [--flavor=FLAVOR] [--network=NETWORK] [--public] [--secgroup=SECGROUPs] [--group=GROUPs] [--key=KEY] [--dryrun] [-v] vm meta list [NAME] vm meta set [NAME] KEY=VALUE... vm meta delete [NAME] KEY... vm script [--name=NAMES] [--username=USERNAME] [--key=KEY] [--dryrun] [--dir=DESTINATION] SCRIPT vm ip assign [NAMES] [--cloud=CLOUD] vm ip show [NAMES] [--group=GROUP] [--cloud=CLOUD] [--output=OUTPUT] [--refresh] vm ip inventory [NAMES] vm ssh [NAMES] [--username=USER] [--quiet] [--ip=IP] [--key=KEY] [--command=COMMAND] vm put SOURCE DESTINATION [NAMES] vm get SOURCE DESTINATION [NAMES] vm rename [OLDNAMES] [NEWNAMES] [--force] [--dryrun] vm wait [--cloud=CLOUD] [--interval=INTERVAL] [--timeout=TIMEOUT] vm info [--cloud=CLOUD] [--output=OUTPUT] vm username USERNAME [NAMES] [--cloud=CLOUD] vm resize [NAMES] [--size=SIZE] Arguments: OUTPUT the output format COMMAND positional arguments, the commands you want to execute on the server(e.g. ls -a) separated by ';', you will get a return of executing result instead of login to the server, note that type in -- is suggested before you input the commands NAME server name. By default it is set to the name of last vm from database. NAMES server name. By default it is set to the name of last vm from database. KEYPAIR_NAME Name of the vm keypair to be used to create VM. Note this is not a path to key. NEWNAMES New names of the VM while renaming. OLDNAMES Old names of the VM while renaming. Options: -v verbose, prints the dict at the end --output=OUTPUT the output format -H --modify-knownhosts Do not modify ~/.ssh/known_hosts file when ssh'ing into a machine --username=USERNAME the username to login into the vm. If not specified it will be guessed from the image name and the cloud --ip=IP give the public ip of the server --cloud=CLOUD give a cloud to work on, if not given, selected or default cloud will be used --count=COUNT give the number of servers to start --detail for table, a brief version is used as default, use this flag to print detailed table --flavor=FLAVOR give the name or id of the flavor --group=GROUP give the group name of server --secgroup=SECGROUP security group name for the server --image=IMAGE give the name or id of the image --key=KEY specify a key to use, input a string which is the full path to the private key file --keypair_name=KEYPAIR_NAME Name of the vm keypair to be used to create VM. Note this is not a path to key. --user=USER give the user name of the server that you want to use to login --name=NAME give the name of the virtual machine --force rename/ delete vms without user's confirmation --command=COMMAND specify the commands to be executed Description: commands used to boot, start or delete servers of a cloud vm default [options...] Displays default parameters that are set for vm boot either on the default cloud or the specified cloud. vm boot [options...] Boots servers on a cloud, user may specify flavor, image .etc, otherwise default values will be used, see how to set default values of a cloud: cloud help vm start [options...] Starts a suspended or stopped vm instance. vm stop [options...] Stops a vm instance . vm delete [options...] Delete servers of a cloud, user may delete a server by its name or id, delete servers of a group or servers of a cloud, give prefix and/or range to find servers by their names. Or user may specify more options to narrow the search vm floating_ip_assign [options...] assign a public ip to a VM of a cloud vm ip show [options...] show the ips of VMs vm ssh [options...] login to a server or execute commands on it vm list [options...] same as command "list vm", please refer to it vm status [options...] Retrieves status of last VM booted on cloud and displays it. vm refresh [--cloud=CLOUDS] this command refreshes the data for virtual machines, images and flavors for the specified clouds. vm ping [NAMES] [--cloud=CLOUDS] [--count=N] [--processors=PROCESSORS] pings the specified virtual machines, while using at most N pings. The ping is executed in parallel. If names are specifies the ping is restricted to the given names in parameter format. If clouds are specified, names that are not in these clouds are ignored. If the name is set in the variables this name is used. cms vm ssh --command=\"uname -a\" executes the uname command on the last booted vm vm script [--name=NAMES] [--username=USERNAME] [--key=KEY] [--dryrun] [--dir=DESTINATION] [--shell=SHELL] SCRIPT The script command copies a shell script to the specified vms into the DESTINATION directory and than execute it. With SHELL you can set the shell for executing the command, this coudl even be a python interpreter. Examples for SHELL are /bin/sh, /usr/bin/env python vm put SOURCE DESTINATION [NAMES] puts the file defined by SOURCE into the DESINATION folder on the specified machines. If the file exists it is overwritten, so be careful. vm get SOURCE DESTINATION [NAMES] gets the file defined by SOURCE into the DESINATION folder on the specified machines. The SOURCE is on the remote machine. If one machine is specified, the SOURCE is the same name as on the remote machine. If multiple machines are specified, the name of the machine will be a prefix to the filename. If the filenames exists, they will be overwritten, so be careful. Tip: give the VM name, but in a hostlist style, which is very convenient when you need a range of VMs e.g. sample[1-3] => ['sample1', 'sample2', 'sample3'] sample[1-3,18] => ['sample1', 'sample2', 'sample3', 'sample18'] Quoting commands: cm vm login gregor-004 --command=\"uname -a\" Limitations: Azure: rename is not supported """ map_parameters(arguments, 'active', 'cloud', 'command', 'dryrun', 'flavor', 'force', 'group' 'output', 'group', 'image', 'interval', 'timeout', 'ip', 'key', 'modify-knownhosts', 'n', 'name', 'public', 'quiet', 'secgroup', 'size', 'username', 'output', 'count', 'network', 'refresh') variables = Variables() database = CmDatabase() arguments.output = Parameter.find("output", arguments, variables, "table") arguments.refresh = Parameter.find_bool("refresh", arguments, variables) if (arguments.meta and arguments.list): name = arguments.NAME if arguments.NAME is None: name = variables['vm'] if name is None: Console.error("No vm specified") cloud = "chameleon" # cloud = Parameter.find(arguments, variables) print(f"vm metadata for {name} on {cloud}") provider = Provider(name=cloud) r = provider.get_server_metadata(name) print(r) elif arguments.meta and arguments.set: metadata = {} pairs = arguments['KEY=VALUE'] for pair in pairs: key, value = pair.split("=", 1) metadata[key] = value name = arguments.NAME if arguments.NAME is None: name = variables['vm'] if name is None: Console.error("No vm specified") cloud = "chameleon" # cloud = Parameter.find(arguments, variables) print(f"cloud {cloud} {name}") provider = Provider(name=cloud) provider.set_server_metadata(name, **metadata) r = provider.get_server_metadata(name) pprint(r) elif arguments.meta and arguments.delete: metadata = {} keys = arguments['KEY'] name = arguments.NAME if arguments.NAME is None: name = variables['vm'] if name is None: Console.error("No vm specified") cloud = "chameleon" # cloud = Parameter.find(arguments, variables) print(f"cloud {cloud} {name}") provider = Provider(name=cloud) for key in keys: provider.delete_server_metadata(name, key) r = provider.get_server_metadata(name) pprint(r) elif arguments.list and arguments.refresh: names = [] clouds, names = Arguments.get_cloud_and_names( "list", arguments, variables) for cloud in clouds: print(f"cloud {cloud}") provider = Provider(name=cloud) vms = provider.list() provider.Print(vms, output=arguments.output, kind="vm") return "" elif arguments.list: names = [] clouds, names = Arguments.get_cloud_and_names( "list", arguments, variables) try: for cloud in clouds: print(f"List {cloud}") p = Provider(cloud) kind = p.kind collection = "{cloud}-vm".format(cloud=cloud, kind=p.kind) db = CmDatabase() vms = db.find(collection=collection) p.Print(vms, output=arguments.output, kind="vm") except Exception as e: Console.error("Error in listing ", traceflag=True) VERBOSE(e) return "" elif arguments.ping: """ vm ping [NAMES] [--cloud=CLOUDS] [--count=N] """ if arguments.NAMES: variables['vm'] = arguments.NAMES if arguments['--cloud']: variables['cloud'] = arguments['--cloud'] clouds, names = Arguments.get_cloud_and_names( "status", arguments, variables) count = arguments.count if arguments.count: count = int(count) else: count = 1 def get_ips(): ips = [] for cloud in clouds: params = {} # gets public ips from database cursor = database.db[f'{cloud}-vm'] for name in names: for node in cursor.find({'name': name}): ips.append(node['ip_public']) ips = list(set(ips)) pprint(ips) return ips ips = get_ips() if len(ips) == 0: Console.warning("no public ip found.") for cloud in clouds: print(f"refresh for cloud {cloud}") provider = Provider(name=cloud) vms = provider.list() ips = get_ips() if len(ips) == 0: Console.error("No vms with public IPS found.") Console.error(" Make sure to use cms vm list --refresh") for ip in ips: result = Shell.ping(host=ip, count=count) banner(ip) print(result) print() elif arguments.check: raise NotImplementedError """ vm check [NAMES] [--cloud=CLOUDS] [--username=USERNAME] """ """ THIS IS ALL WRONG AS PROVIDER DEPENDENT !!! if arguments.NAMES: variables['vm'] = arguments.NAMES if arguments['--cloud']: variables['cloud'] = arguments['--cloud'] clouds, names = Arguments.get_cloud_and_names("status", arguments, variables) for cloud in clouds: provider = Provider(cloud) params = {} params['key'] = \ provider.p.spec["credentials"]['EC2_PRIVATE_KEY_FILE_PATH'] + \ provider.p.spec["credentials"]['EC2_PRIVATE_KEY_FILE_NAME'] params['username'] = arguments['--username'] # or get from db processors = arguments['--processors'] if processors: params['processors'] = int(processors[0]) # gets public ips from database public_ips = [] cursor = database.db['{cloud}-vm'] for name in names: for node in cursor.find({'name': name}): public_ips.append(node['public_ips']) public_ips = [y for x in public_ips for y in x] Host.check(hosts=public_ips, **params) """ elif arguments.status: if arguments.NAMES: variables['vm'] = arguments.NAMES if arguments['--cloud']: variables['cloud'] = arguments['--cloud'] clouds, names = Arguments.get_cloud_and_names( "status", arguments, variables) # gets status from database for cloud in clouds: provider = Provider(cloud) status = [] cursor = database.db[f'{cloud}-vm'] print(cloud) for name in names: for node in cursor.find({'name': name}): status.append(node) provider.Print(status, output=arguments.output, kind="status") return "" elif arguments.start: # TODO: not tested if arguments.NAMES: names = variables['vm'] = arguments.NAMES if arguments['--cloud']: variables['cloud'] = arguments['--cloud'] clouds, names = Arguments.get_cloud_and_names( "stop", arguments, variables) cloud = clouds[0] print(cloud) print(names) for name in names: provider = Provider(cloud) if arguments['--dryrun']: print(f"start node {name}") else: vms = provider.start(name=name, cloud=cloud) provider.Print(vms, output=arguments.output, kind="vm") return "" elif arguments.stop: # TODO: not tested if arguments.NAMES: variables['vm'] = arguments.NAMES if arguments['--cloud']: variables['cloud'] = arguments['--cloud'] clouds, names = Arguments.get_cloud_and_names( "stop", arguments, variables) for cloud in clouds: params = {} provider = Provider(cloud) if arguments['--dryrun']: Console.ok(f"Dryrun stop: " f" {cloud}\n" f" {names}" f" {provider}") else: for name in names: vms = provider.stop(name) provider.Print(vms, output=arguments.output, kind="vm") elif arguments.terminate: # TODO: not tested if arguments.NAMES: variables['vm'] = arguments.NAMES if arguments['--cloud']: variables['cloud'] = arguments['--cloud'] clouds, names = Arguments.get_cloud_and_names( "stop", arguments, variables) for cloud in clouds: params = {} provider = Provider(cloud) if arguments['--dryrun']: Console.ok(f"Dryrun terminate: " f" {cloud}\n" f" {names}" f" {provider}") else: for name in names: vms = provider.destroy(name) provider.Print(vms, output=arguments.output, kind="vm") elif arguments.delete: if arguments.NAMES: variables['vm'] = arguments.NAMES if arguments['--cloud']: variables['cloud'] = arguments['--cloud'] clouds, names = Arguments.get_cloud_and_names( "stop", arguments, variables) if names is not None: pass elif clouds is not None: for cloud in clouds: provider = Provider(cloud) vms = provider.list() for vm in vms: r = provider.destroy(name=vm) return "" else: return "" for cloud in clouds: provider = Provider(cloud) vms = provider.list() for vm in vms: name = vm["cm"]["name"] if name in names: r = provider.destroy(name=name) # TODO: username, secgroup elif arguments.boot: # not everything works """ vm boot [--name=VMNAMES] [--cloud=CLOUD] [--username=USERNAME] [--image=IMAGE] [--flavor=FLAVOR] [--network=NETWORK] [--public] [--secgroup=SECGROUP] [--key=KEY] [--group=GROUP] [--dryrun] """ # for name in names: # node = p.create(name=name, size=flavor, image=image) # VERBOSE(arguments) parameters = dotdict() names = Parameter.expand(arguments.name) cloud = Parameter.find("cloud", arguments, variables.dict()) defaults = Config()[f"cloudmesh.cloud.{cloud}.default"] groups = Parameter.find("group", arguments, variables.dict(), {"group": "default"}) parameters = dotdict() # parameters.names = arguments.name parameters.group = groups for attribute in [ "image", "username", "flavor", "key", "network", "secgroup" ]: parameters[attribute] = Parameter.find(attribute, arguments, variables.dict(), defaults) if arguments.username is None: parameters.user = Image.guess_username(parameters.image) provider = Provider(name=cloud) parameters.secgroup = arguments.secgroup or "default" # # determine names # if names and arguments.n and len(names) > 1: Console.error( f"When using --n={arguments.n}, you can only specify one name" ) return "" # cases # # only name --name = "a[1,2]" # name and count # --name="a" --n=3, names must be of length 1 # only count --n=2 names are read form var # nothing, just use one vm # determin names _names = [] if not names: if not arguments.n: count = 1 else: count = int(arguments.n) for i in range(0, count): if names is None: n = Name() n.incr() name = str(n) else: n = names[i] name = str(n) _names.append(name) names = _names elif len(names) == 1 and arguments.n: name = names[0] for i in range(0, int(arguments.n)): _names.append(f"{name}-{i}") names = _names # pprint(parameters) for name in names: parameters.name = name if arguments['--dryrun']: banner("boot") pprint(parameters) Console.ok(f"Dryrun boot {name}: \n" f" cloud={cloud}\n" f" names={names}\n" f" provider={provider}") print() for attribute in parameters: value = parameters[attribute] Console.ok(f" {attribute}={value}") else: # parameters.progress = len(parameters.names) < 2 try: vms = provider.create(**parameters) except TimeoutError: Console.error( f"Timeout during vm creation. There may be a problem with the cloud {cloud}" ) except Exception as e: Console.error("create problem", traceflag=True) print(e) return "" variables['vm'] = str(n) if arguments["-v"]: banner("Details") pprint(vms) # provider.Print(arguments.output, "vm", vms) elif arguments.info: """ vm info [--cloud=CLOUD] [--output=OUTPUT] """ print("info for the vm") cloud, names = Arguments.get_cloud_and_names( "info", arguments, variables) raise NotImplementedError elif arguments.rename: raise NotImplementedError # Not tested print("rename the vm") v = Variables() cloud = v["cloud"] p = Provider(cloud) try: oldnames = Parameter.expand(arguments["OLDNAMES"]) newnames = Parameter.expand(arguments["NEWNAMES"]) force = arguments["--force"] if oldnames is None or newnames is None: Console.error("Wrong VMs specified for rename", traceflag=False) elif len(oldnames) != len(newnames): Console.error("The number of VMs to be renamed is wrong", traceflag=False) else: print(oldnames) print(newnames) for i in range(0, len(oldnames)): oldname = oldnames[i] newname = newnames[i] if arguments["--dryrun"]: Console.ok("Rename {} to {}".format( oldname, newname)) else: print(f"rename {oldname} -> {newname}") p.rename(source=oldname, destination=newname) msg = "info. OK." Console.ok(msg) except Exception as e: Error.traceback(e) Console.error("Problem renaming instances", traceflag=True) elif arguments["ip"] and arguments["show"]: raise NotImplementedError print("show the ips") """ vm ip show [NAMES] [--group=GROUP] [--cloud=CLOUD] [--output=OUTPUT] [--refresh] """ elif arguments["ip"] and arguments["assign"]: raise NotImplementedError """ vm ip assign [NAMES] [--cloud=CLOUD] """ print("assign the public ip") elif arguments["ip"] and arguments["inventory"]: raise NotImplementedError """ vm ip inventory [NAMES] """ print("list ips that could be assigned") elif arguments.default: raise NotImplementedError print("sets defaults for the vm") elif arguments.script: raise NotImplementedError clouds, names = Arguments.get_cloud_and_names( "run", arguments, variables) username = arguments['--username'] script = arguments.SCRIPT for cloud in clouds: provider = Provider(cloud) name_ips = {} cursor = database.db['{}-node'.format(cloud)] for name in names: for node in cursor.find({'name': name}): name_ips[name] = node['public_ips'] if arguments['--dryrun']: print("run script {} on vms: {}".format(script, names)) else: provider.ssh(name_ips, username=username, script=script) elif arguments.username: raise NotImplementedError """ vm username USERNAME [NAMES] [--cloud=CLOUD] """ print("sets the username for the vm") elif arguments.resize: raise NotImplementedError """ vm resize [NAMES] [--size=SIZE] """ pass elif arguments.ssh: """ vm ssh [NAMES] [--username=USER] [--quiet] [--ip=IP] [--key=KEY] [--command=COMMAND] """ # VERBOSE(arguments) clouds, names, command = Arguments.get_commands( "ssh", arguments, variables) # print (clouds) # print(names) # print (command) if arguments.command is None and len(names) > 1: Console.error("Interactive shell can only be done on one vm") return "" elif arguments.command is None and len(names) == 1: name = names[0] cloud = clouds[0] cm = CmDatabase() try: vm = cm.find_name(name, "vm")[0] except IndexError: Console.error(f"could not find vm {name}") return "" # VERBOSE(vm) cloud = vm["cm"]["cloud"] provider = Provider(name=cloud) try: provider.ssh(vm=vm) except KeyError: vms = provider.list() provider.Print(vms, output=arguments.output, kind="vm") provider.ssh(vm=vm) return "" else: # command on all vms if clouds is None or names is None or command is None: return "" else: for cloud in clouds: p = Provider(cloud) for name in names: cm = CmDatabase() try: vm = cm.find_name(name, "vm")[0] except IndexError: Console.error(f"could not find vm {name}") continue r = p.ssh(vm=vm, command=command) print(r) return "" elif arguments.console: # why is this not vm clouds, names, command = Arguments.get_commands( "ssh", arguments, variables) print(clouds) print(names) print(command) for cloud in clouds: p = Provider(cloud) for name in names: cm = CmDatabase() try: vm = cm.find_name(name, "vm")[0] except IndexError: Console.error(f"could not find vm {name}") continue r = p.console(vm=vm) print(r) return "" elif arguments.log: # why is this not vm clouds, names, command = Arguments.get_commands( "ssh", arguments, variables) print(clouds) print(names) print(command) for cloud in clouds: p = Provider(cloud) for name in names: cm = CmDatabase() try: vm = cm.find_name(name, "vm")[0] except IndexError: Console.error(f"could not find vm {name}") continue r = p.log(vm=vm) print(r) return "" elif arguments.wait: """ vm wait [--cloud=CLOUD] [--interval=INTERVAL] [--timeout=TIMEOUT] """ # why is this not vm clouds, names, command = Arguments.get_commands( "ssh", arguments, variables) # print (clouds) # print (names) # print (command) for cloud in clouds: p = Provider(cloud) for name in names: cm = CmDatabase() try: vm = cm.find_name(name, "vm")[0] except IndexError: Console.error(f"could not find vm {name}") continue r = p.wait(vm=vm, interval=arguments.interval, timeout=arguments.timeout) if r: Console.ok("Instance available for SSH") else: Console.error( f"Instance unavailable after timeout of {arguments.timeout}" ) # print(r) return "" elif arguments.put: """ vm put SOURCE DESTINATION """ clouds, names, command = Arguments.get_commands( "ssh", arguments, variables) key = variables['key'] source = arguments['SOURCE'] destination = arguments['DESTINATION'] for cloud in clouds: p = Provider(name=cloud) cm = CmDatabase() for name in names: try: vms = cm.find_name(name, "vm") except IndexError: Console.error(f"could not find vm {name}") return "" # VERBOSE(vm) for vm in vms: try: ip = vm['public_ips'] except: try: ip = p.get_public_ip(name=name) except: Console.error( f"could not find a public ip for vm {name}", traceflag=True) return Console.error( f"could not find a public ip for vm {name}", traceflag=True) return # get the username try: # username not in vm...guessing imagename = list( cm.collection(cloud + '-image').find( {'ImageId': vm['ImageId']}))[0]['name'] print(imagename) user = Image.guess_username(image=imagename, cloud=cloud) except: try: user = vm['os_profile']['admin_username'] except: Console.error( f"could not find a valid username for " f"{name}, try refreshing the image list", traceflag=True) return Console.error( f"could not find a valid username for {name}, try refreshing the image list" ) return cmd = f'scp -i {key} {source} {user}@{ip}:{destination}' print(cmd) os.system(cmd) return ""
from cloudmesh.mongo.CmDatabase import CmDatabase cmdb = CmDatabase() cmdb.connect() cluster_virtual = cmdb.collection("cluster-virtual") print(cluster_virtual) cluster_virtual.insert_many([{"hello":"world", "test":1}, {"hello":"w", "test":2}])
def get_aws_pricing(refresh=False): # connect to the db cm = CmDatabase() # check to see if AWS frugal entries already exist awsinfo = cm.collection('aws-frugal') if awsinfo.estimated_document_count() > 0 and not refresh: # frugal information alredy exists, so return it Console.msg(f"Using local db aws flavors...") return awsinfo # helper function to parse aws info def aws_parse(input): if ('location' not in x['attributes'] or 'vcpu' not in x[ 'attributes'] or 'price' not in x or x['attributes'][ 'memory'] == 'NA'): return return np.array(['aws', x['sku'], x['attributes']['location'], float(x['attributes']['vcpu']), float(x['attributes']['memory'][:-4].replace(',', '')), float(x['price']['pricePerUnit']['USD'])]) # check to see if general flavor entries exist awsinfo = cm.collection('aws-flavor') aws_list = [] if awsinfo.estimated_document_count() > 0 and not refresh: # can get info directly from db, just need to reshape Console.msg(f"Calculating with local db aws flavors...") for x in awsinfo.find(): tempray = aws_parse(x) if tempray is not None: aws_list.append(aws_parse(x)) else: Console.msg(f"refreshing aws flavors in now ...") try: awsprovider = awsprv.Provider( name='aws', configuration="~/.cloudmesh/cloudmesh.yaml") except: Console.msg("No aws credentials") return awsinfo = awsprovider.flavors() for x in awsinfo: tempray = aws_parse(x) if tempray is not None: aws_list.append(aws_parse(x)) awsinfo = np.stack(aws_list, axis=0) awsinfo = helpers.format_mat(awsinfo) # convert to list of dicts awsinfo = awsinfo.to_dict('records') # write back to cm db for entry in awsinfo: entry["cm"] = { "kind": 'frugal', "driver": 'aws', "cloud": 'aws', "name": str(entry['machine-name'] + '-' + entry['location']), "updated": str(datetime.utcnow()), } Console.msg(f"Writing back to db ...") cm.update(awsinfo, progress=True) return cm.collection('aws-frugal')
def list(self, order='price', resultssize=25, refresh=False, printit=True, benchmark=False, cloud=None): clouds = ['aws', 'azure', 'gcp'] if cloud in clouds: clouds = [cloud] if benchmark: # get benchmarks cm = CmDatabase() benchmarks = [] for cloud in clouds: print("searching " + cloud) benchmarktemp = list( cm.collection(cloud + '-frugal-benchmark').find()) benchmarks = benchmarks + benchmarktemp print( Printer.write(benchmarks, order=[ 'cloud', 'name', 'region', 'ImageId', 'flavor', 'updated', 'BenchmarkTime' ])) return else: #check to make sure that order is either price, cores, or memory if order not in ['price', 'cores', 'memory']: Console.error( f'order argument must be price, cores, or memory') return printlist = [] if 'aws' in clouds: # get aws pricing info printlist = printlist + list( aws_frugal.get_aws_pricing(refresh=refresh).find()) if 'gcp' in clouds: # get gcp pricing info printlist = printlist + list( gcp_frugal.get_google_pricing(refresh=refresh).find()) if 'azure' in clouds: # get azure pricing info printlist = printlist + list( azure_frugal.get_azure_pricing(refresh=refresh).find()) # turn numpy array into a pandas dataframe, assign column names, and remove na values flavor_frame = pd.DataFrame(printlist)[[ 'provider', 'machine-name', 'location', 'cores', 'core/price', 'memory', 'memory/price', 'price' ]] flavor_frame = flavor_frame.replace([np.inf, -np.inf], np.nan).dropna() # sort the dataframe by order if order == 'cores': flavor_frame = flavor_frame.sort_values(by=['core/price'], ascending=False) elif order == 'memory': flavor_frame = flavor_frame.sort_values(by=['memory/price'], ascending=False) else: flavor_frame = flavor_frame.sort_values(by=['price'], ascending=True) # print out the dataframe if printit, print results limited by resultssize if printit: print( Printer.write( flavor_frame.head(resultssize).to_dict('records'), order=[ 'provider', 'machine-name', 'location', 'cores', 'core/price', 'memory', 'memory/price', 'price' ])) # return the final sorted data frame return flavor_frame
class WorkFlowDB(object): def __init__(self, name="workflow"): self.database = CmDatabase() self.workflow_name = name self.collection = self.database.collection(f"{name}-flow") def attributes(self, name): data = { "cm": { "kind": "flow", "cloud": self.workflow_name, "name": name }, "kind": "flow", "cloud": self.workflow_name, "name": name } return data @DatabaseUpdate() def add_node(self, node): name = node["name"] node.update(self.attributes(name)) return node def add_edge(self, node, depends_on): self.collection.update_one({"name": node}, {"$push": { "dependencies": depends_on }}) def _node_from_db(self, db_obj): reconstructed = Node(db_obj["name"]) reconstructed.workflow = self.workflow_name reconstructed.dependencies = db_obj["dependencies"] reconstructed.status = db_obj.get("status", "pending") return reconstructed def get_node(self, name=None): return self._node_from_db(self.collection.find_one({"name": name})) def list(self, node=None, edge=None): query = {} if node: query["name"] = node if edge: query["dependencies"] = edge return self.collection.find(query) def list_nodes(self): return [self._node_from_db(node) for node in self.list()] def list_edges(self): return self.collection.aggregate([{ "$unwind": "dependecies" }, { "$project": { "to": "$name", "from": "$dependecies" } }]) def list_all_workflows(self): all_colls = self.database.collections() return [ name for name in all_colls if "flow" in name and "active" not in name ] def set_node_status(self, node, status): return self.collection.update_one({"name": node}, {"$set": { "status": status }}) def find_root_nodes(self): root_nodes = self.collection.find({ "dependencies.0": { "$exists": False }, "status": "pending" }) return [self._node_from_db(node) for node in root_nodes] def switch_to_active_flow(self): started_collection = f"{self.workflow_name}-flow-active" self.collection = self.database.collection(started_collection) def resolve_node_dependency(self, name=None): return self.collection.update_many({"dependencies": name}, {"$pull": { "dependencies": name }}) def add_specification(self, spec): pass def start_flow(self): started_collection = f"{self.workflow_name}-flow-active" self.collection.aggregate([{ "$project": { "dependencies": 1, "cm": 1, "kind": 1, "cloud": 1, "name": 1, "status": "pending" } }, { "$out": started_collection }]) self.switch_to_active_flow() def add_graph(self, yamlfile): pass
def do_cluster(self, args, arguments): """ :: Usage: cluster test cluster build --id=ID LABEL cluster create --cloud=CLOUD --n=N LABEL cluster add --id="[ID]" --all LABEL cluster remove --id="[ID]" --all LABEL cluster terminate --all LABEL cluster info [--verbose=V] [LABEL] This command allows you to create and interact with an available cluster of machines. Arguments: ID An existing machine ID to be reserved for the cluster. LABEL The label for the cluster. CLOUD Cloud platform to initialize VMs. N Number of instances to request. V Verbosity level. Options: --id Specify string containing list of IDs, comma delimited format "id1,id2,...,idx". --cloud Specify cloud platform {AWS, Azure, Openstack}. --n Specify number of VMs to initialize. --all OPTIONAL. Overrides --id, will pass all machines as an argument. --verbose OPTIONAL. Provides verbosity level for info. Description: cluster build --id=ID LABEL Groups together existing machines and reserves them for cluster use. Pass a comma-delimited list of machine ID's as a string. Pass --all to associate all available machines to cluster. cluster create --cloud=CLOUD --n=N LABEL Automatically requests VMs from the cloud service requested. cluster add --id="[ID]" --all LABEL Adds given machine IDs to cluster. Pass --all to associate all available machines to cluster. cluster remove --id="[ID]" LABEL Removes given machine IDs from cluster. Pass --all to disassociate all machines from cluster. cluster terminate --all LABEL Terminates all instances associated with the cluster, wipes cluster data. If --all is passed, terminates all running clusters. cluster info --all --verbose=v [LABEL] Retrieves cluster data and machine data associatred with cluster. Verbosity level 1 provides high-level cluster information and list of machines associated. Verbosity level 2 provides cluster information, machine information and status. Verbosity level 3 provides all available information. """ map_parameters(arguments, 'id', 'label', 'cloud', 'n', 'v', 'all', 'verbose') # inv = Inventory() # inv.read() config = Config() user = config["cloudmesh.profile.user"] s = Shell() cmdb = CmDatabase() if arguments.test: cmdb = CmDatabase() virtual_clusters = cmdb.collection("cluster-virtual") print(*[index for index in virtual_clusters.list_indexes()]) if arguments.build: ids, label = arguments.id, arguments.LABEL # Builds and stores a cluster connected to existing machine ids machines = ids.split(",") cluster_machines = [] for i, machine in enumerate(machines): cluster_machines.append({ f"{machine}_{i}": { "type": "cloud", "cloud": None, "status": "available", "deployment": None } }) print(f"Adding the following machines to cluster-cloud {label}: ") VERBOSE(cluster_machines) collection = cmdb.collection("cluster-cloud") collection.insert_one({label: cluster_machines}) # # TODO Revise to update to correct mongo create/update # cmdb.UPDATE(clusters) if arguments.create: n, label, cloud = arguments.n, arguments.label, arguments.cloud ids = [f"label_{i}" for i in range(n)].join(",") starting = [ s.run(f"cms vm boot --name={id} --cloud={cloud}") for id in ids ] s.run(f"cms cluster build --id={ids} {label}") print(f"Starting {starting}") elif arguments.add: pass elif arguments.remove: pass elif arguments.terminate: pass elif arguments.info: v, label = arguments.verbose or arguments.v or None, arguments.LABEL or None if label: print(f"Searching for {label}") virtual_clusters, cloud_clusters = cmdb.collection( "cluster-virtual"), cmdb.collection("cluster-cloud") output = { 'virtual': [i for i in virtual_clusters.find(label)], 'cloud': [i for i in cloud_clusters.find(label)] } print(output) return ""
class Provider(object): # broken kind = "volume" @staticmethod def get_kind(): """ Get the kind of provider :return: string """ kind = ["multipass", "aws", "azure", "google", "openstack", "oracle"] return kind @staticmethod def get_provider(kind): """ Get the provider with specific provider (kind) :param kind: :return: """ if kind == "multipass": from cloudmesh.volume.multipass.Provider import Provider as P elif kind == "aws": from cloudmesh.volume.aws.Provider import Provider as P elif kind == "azure": from cloudmesh.volume.azure.Provider import Provider as P elif kind == "google": from cloudmesh.volume.google.Provider import Provider as P elif kind == "openstack": from cloudmesh.volume.openstack.Provider import Provider as P elif kind == "oracle": from cloudmesh.volume.oracle.Provider import Provider as P else: Console.error(f"Compute provider {kind} not supported") raise ValueError(f"Compute provider {kind} not supported") return P # noinspection PyPep8Naming def Print(self, data, kind=None, output="table"): """ Print out the result dictionary as table(by default) or json. :param data: dic returned from volume functions :param kind: kind of provider :param output: "table" or "json" :return: """ if kind is None and len(data) > 0: kind = data[0]["cm"]["kind"] if output == "table": order = self.provider.output[kind]['order'] header = self.provider.output[kind]['header'] if 'humanize' in self.provider.output[kind]: humanize = self.provider.output[kind]['humanize'] else: humanize = None print(Printer.flatwrite(data, sort_keys=["name"], order=order, header=header, output=output, humanize=humanize) ) else: print(Printer.write(data, output=output)) def __init__(self, name=None, configuration="~/.cloudmesh/cloudmesh.yaml"): try: conf = Config(configuration)["cloudmesh"] self.spec = conf["volume"][name] self.cloud = name self.kind = self.spec["cm"]["kind"] super().__init__() except: Console.error(f"provider {name} not found in {configuration}") raise ValueError(f"provider {name} not found in {configuration}") P = None if self.kind in ["multipass", "aws", "azure", "google", "openstack", "oracle"]: P = Provider.get_provider(self.kind) if P is None: Console.error(f"provider {name} not supported") raise ValueError(f"provider {name} not supported") self.provider = P(self.cloud) @DatabaseUpdate() def create(self, **kwargs): """ Create a volume. :param name (string): name of volume :param region (string): availability-zone :param size (integer): size of volume :param volume_type (string): type of volume. :param description (string) :return: dict """ try: data = self.provider.create(**kwargs) variables = Variables() variables["volume"] = data[0]["cm"]["name"] except: raise ValueError("Volume could not be created") return data @DatabaseUpdate() def delete(self, name=None): """ Delete volumes. If name is not given, delete the most recent volume. :param name: List of volume name :return: """ d = self.provider.delete(name) return d @DatabaseUpdate() def list(self, **kwargs): """ This command list all volumes as follows: If names are given, search through all the active clouds and list all the volumes. If names and cloud are given, list all volumes under the cloud. If cloud is given, list all the volumes under the cloud. If cloud is not given, list all the volumes under current cloud. If vm is given, under the current cloud, list all the volumes attaching to the vm. If region is given, under the current cloud, list all volumes in that region. :param names: List of volume names :param vm: The name of the virtual machine :param region: The name of the region :param cloud: The name of the cloud :param refresh: If refresh the information is taken from the cloud :return: dict """ data = self.provider.list(**kwargs) return data def info(self, name=None): """ Search through the list of volumes, find the matching volume with name, return the dict of matched volume :param name: volume name to match :return: dict """ volumes = self.provider.list() for volume in volumes: if volume["cm"]["name"] == name: return volume return None def search(self, name=None): """ Calls info(self, name=None) :param name: volume name to match :return: dict """ return self.info(name=name) @DatabaseUpdate() def status(self, name=None): """ This function returns status of volume, such as "available", "in-use" and etc.. :param name: name of volume :return: string """ volume_status = self.provider.status(name) return volume_status @DatabaseUpdate() def attach(self, names=None, vm=None): """ Attach volume to a vm. If names is not specified, attach the most recent volume to vm. :param name: volume name :param vm: vm name which the volume will be attached to :return: dict """ result = self.provider.attach(names, vm) return result @DatabaseUpdate() def detach(self, name=None): """ Detach volumes from vm. If success, the last volume will be saved as the most recent volume. :param names: names of volumes to detach :return: dict """ try: result = self.provider.detach(name) variables = Variables() variables["volume"] = result["cm"]["name"] except: raise ValueError("Volume could not be detached") return result @DatabaseUpdate() def add_tag(self, **kwargs): """ This function add tag to a volume. If name is not specified, then tag will be added to the last volume. If success, the volume will be saved as the most recent volume. :param name: name of volume :param kwargs: key: name of tag value: value of tag :return: self.list() """ try: result = self.provider.add_tag(**kwargs) variables = Variables() variables["volume"] = result["cm"]["name"] except: raise ValueError("Tag could not be added") return result @DatabaseUpdate() def migrate(self, **kwargs): """ Migrate volume from one vm to another vm in the same cloud service. :param name (string): the volume name :param vm (string): the vm name :return: dict """ try: result = self.provider.migrate(**kwargs) except: raise ValueError("Volume could not be migrate") return result @DatabaseUpdate() def sync(self, **kwargs): """ synchronize one volume with another volume in the same cloud service. :param names (list): list of volume names :return: dict """ try: result = self.provider.sync(**kwargs) except: raise ValueError("Volume could not be synchronized") return result @DatabaseUpdate() def purge(self, **kwargs): """ purge deleted volumes in MongoDB database :return: dict """ collection = f"{self.cloud}-volume" self.cm = CmDatabase() if self.cloud == 'aws' or self.cloud == 'multipass': self.cm.collection(collection).delete_many({"State": "deleted"}) elif self.cloud == 'oracle': self.cm.collection(collection).delete_many( {"lifecycle_state": "deleted"}) elif self.cloud == 'azure': self.cm.collection(collection).delete_many( {"disk_state": "deleted"}) elif self.cm.cloud == 'google' or self.cloud == 'openstack': self.cm.collection(collection).delete_many({"status": "deleted"}) return self.provider.list()