class Vc(ListResource):
cm = CloudmeshDatabase()
'''
vc key list NAMES [--usort]
'''
@classmethod
def list(cls, names, usort=False, format="table"):
"""
This method lists all vcs of the cloud
:param cloud: the cloud name
"""
Console.error("NOT YET IMPLEMENTED")
return None
class Hpc(ListResource):
cm = CloudmeshDatabase()
@classmethod
def refresh(cls, cloud):
"""
This method would refresh the hpc list by first clearing
the database, then inserting new data
:param cloud: the cloud name
"""
return cls.cm.refresh(kind='hpc', category=cloud)
@classmethod
def list(cls, cloud, live=False, format="table"):
"""
This method lists all hpcs of the cloud
:param cloud: the cloud name
"""
try:
if live:
cls.refresh(cloud)
elements = cls.cm.find(kind="hpc", category=cloud)
# pprint(elements)
(order, header) = CloudProvider(cloud).get_attributes("hpc")
return Printer.write(elements,
order=order,
header=header,
output=format)
except Exception as ex:
Console.error(ex.message)
@classmethod
def details(cls, cloud, id, live=False, format="table"):
if live:
cls.refresh(cloud)
return CloudProvider(cloud).details('hpc', cloud, id, format)
class Counter(object):
"""
A counter is used to keep track of some value that can be increased
and is associated with a user. Typically it is used to increment the
vm id or the job id.
"""
cm = CloudmeshDatabase()
@classmethod
def incr(cls, name='counter', user=None):
"""
increments the counter by one
:param name: name of the counter
:param user: username associated with the counter
:return:
"""
cls.cm.counter_incr(name=name, user=user)
@classmethod
def get(cls, name='counter', user=None):
"""
returns the value of the counter
:param name: name of the counter
:param user: username associated with the counter
:return: the value of the counter
"""
return cls.cm.counter_get(name=name, user=user)
@classmethod
def set(cls, name='counter', value=None, user=None):
"""
sets a counter associated with a particular user
:param name: name of the counter
:param user: username associated with the counter
:param value: the value
:return:
"""
cls.cm.counter_set(name=name, value=value, user=user)
import json
import os.path
from .ClusterCommand2 import Command as ClusterCommand
from .StackCommand import Command as StackCommand
from cloudmesh_client.cloud.stack import BigDataStack, ProjectDB, \
ProjectFactory, SanityCheckError
from cloudmesh_client.common.dotdict import dotdict
from cloudmesh_client.db import CloudmeshDatabase, SPECIFICATION
from cloudmesh_client.default import Default, Names
from cloudmesh_client.shell.command import CloudPluginCommand, PluginCommand, \
command
from cloudmesh_client.shell.console import Console
db = CloudmeshDatabase()
class Command(object):
def sync(self, stackname=None):
name = stackname or Default.active_stack
try:
spec = db.select(SPECIFICATION, type='stack', name=name)[0]
except IndexError:
Console.error('No project defined. Use `cm hadoop define` first')
return
kwargs = spec.get()
path = kwargs.pop('local_path')
class Image(ListResource):
cm = CloudmeshDatabase()
@classmethod
def refresh(cls, cloud):
"""
This method would refresh the image list by first clearing
the database, then inserting new data
:param cloud: the cloud name
"""
# Newly implemented refresh
result = cls.cm.refresh("image", cloud)
return result
@classmethod
def list(cls, cloud, format="table"):
"""
This method lists all images of the cloud
:param cloud: the cloud name
"""
# TODO: make a CloudmeshDatabase without requiring the user=
try:
elements = cls.cm.find(kind="image", category=cloud, scope="all")
(order, header) = CloudProvider(cloud).get_attributes("image")
return Printer.write(elements,
order=order,
header=header,
output=format)
except Exception as ex:
Console.error(ex.message)
@classmethod
def details(cls, cloud, id, live=False, format="table"):
if live:
cls.refresh(cloud)
return CloudProvider(cloud).details('image', cloud, id, format)
@classmethod
def guess_username_from_category(cls, category, image, username=None):
chameleon = "chameleon" in ConfigDict(
filename="cloudmesh.yaml"
)["cloudmesh"]["clouds"][category]["cm_host"]
username = None
if chameleon:
username = "******"
else:
if username is None:
Console.error("Could not guess the username of the vm",
traceflag=False)
return
username = username or Image.guess_username(image)
return username
@classmethod
def guess_username(cls, vm_name, cloud=None, description=None):
username = None
names = [vm_name]
if description is not None:
names.append(description)
chameleon = cloud == "chameleon"
for name in names:
name = name.lower()
if name.startswith("cc-") or chameleon:
username = "******"
break
elif any(x in name for x in ["ubuntu", "wily", "xenial"]):
username = "******"
break
elif "centos" in name:
username = "******"
break
elif "fedora" in name:
username = "******"
break
elif "rhel" in name:
username = "******"
break
elif "cirros" in name:
username = "******"
break
elif "coreos" in name:
username = "******"
break
return username
@classmethod
def get(cls, name=None, cloud=None):
cloud = cloud or Default.cloud
name = name or Default.image
image = cls.cm.find(kind="image",
category=cloud,
name=name,
output='dict',
scope='first')
return image
@classmethod
def get_username(cls, name, cloud, guess=False):
image = cls.get(cloud=cloud, name=name)
if guess and image.username is None:
return cls.guess_username(image.name)
return image.username
@classmethod
def set_username(cls, name=None, cloud=None, username=None):
image = cls.get(cloud=cloud, name=name)
cls.cm.set(name,
"username",
username,
provider=image.provider,
kind="image",
scope="first")
class Reservation(ListResource):
cm = CloudmeshDatabase()
def info(cls, user=None, project=None):
"""
prints if the user has access to the reservation an on which host.
:param user:
:param project:
:return:
"""
TODO.implement()
def add_from_file(cls, filename):
"""
:param filename:
:return:
"""
TODO.implement()
@classmethod
def add(cls,
name,
start,
end,
user=None,
project=None,
hosts=None,
description=None,
category=None):
"""
:param name: Name of reservation
:param start: Start time of reservation
:param end: End time of reservation
:param user: Reserved by this user
:param project: Reservation project
:param hosts: Reserved hosts
:param description: Description
:param cloud: Cloud into which reservation done
:return:
"""
o = {
"provider": "general",
"kind": "reservation",
"name": name,
"hosts": hosts,
"start": start,
"end": end,
"description": description,
"category": category,
"user": user,
"project": project
}
cls.cm.add(o)
@classmethod
def delete(cls,
name=None,
start=None,
end=None,
user=None,
project=None,
hosts=None):
"""
:param name: Name of reservation
:param start: Start time of reservation
:param end: End time of reservation
:param user: Reserved by this user
:param project: Reservation project
:param hosts: Hosts reserved
:return:
"""
args = {}
if name is not None:
args['name'] = name
if start is not None:
args['start_time'] = start
if end is not None:
args['end_time'] = end
if user is not None:
args['user'] = user
if project is not None:
args['project'] = project
if hosts is not None:
args['hosts'] = hosts
# TODO: Improve this logic
result = cls.cm.find(kind="reservation",
provider="general",
output="dict",
scope='all',
**args)
if result is not None:
for res in result:
cls.cm.delete(kind="reservation",
provider="general",
name=res["name"])
@classmethod
def delete_from_file(cls, filename):
"""
:param filename:
:return:
"""
TODO.implement()
@classmethod
def suspend(cls, names=None):
TODO.implement()
@classmethod
def clear(cls, names=None):
TODO.implement()
@classmethod
def refresh(cls, names=None):
TODO.implement()
@classmethod
def resume(cls, names=None):
TODO.implement()
@classmethod
def list(cls,
name=None,
start=None,
end=None,
user=None,
project=None,
hosts=None):
"""
:param name: Name of reservation
:param start: Start time of reservation
:param end: End time of reservation
:param user: Reserved by this user
:param project: Reservation project
:param hosts: Hosts reserved
:return:
"""
args = {}
if name is not None:
args['name'] = name
if start is not None:
args['start_time'] = start
if end is not None:
args['end_time'] = end
if user is not None:
args['user'] = user
if project is not None:
args['project'] = project
if hosts is not None:
args['hosts'] = hosts
args["kind"] = "reservation"
# print(args)
result = cls.cm.find(**args)
# print("RESULT:- {}".format(result))
return result
@classmethod
def update(cls,
name,
start,
end,
user=None,
project=None,
hosts=None,
description=None,
cloud=None):
entry = {
'start_time': start,
'end_time': end,
'name': name,
'user': user,
'project': project,
'hosts': hosts,
'description': description,
'cloud': cloud,
"kind": "reservation",
"provider": "general"
}
update = dict(entry)
for key in entry:
if entry[key] is None:
del update[key]
print("ARGS", update)
cls.cm.add(entry)
class Key(ListResource):
cm = CloudmeshDatabase()
@classmethod
def info(cls, **kwargs):
raise NotImplementedError()
@classmethod
def get_from_dir(cls, directory=None, store=True):
directory = directory or Config.path_expand("~/.ssh")
files = [
file
for file in os.listdir(expanduser(Config.path_expand(directory)))
if file.lower().endswith(".pub")
]
d = []
for file in files:
location = Config.path_expand("{:}/{:}".format(directory, file))
sshkey = SSHkey(location).get()
i = sshkey["comment"]
if i is not None:
i = i.replace("@", "_")
i = i.replace("-", "_")
i = i.replace(" ", "_")
i = i.replace(".", "_")
else:
# use base name
i = file.replace(".pub", "")
sshkey["kind"] = "key"
sshkey["source"] = 'file'
if store:
cls._add_from_sshkey(dict(sshkey),
keyname=sshkey["name"],
source=sshkey["source"],
uri=sshkey["uri"])
else:
d.append(dict(sshkey))
if not store:
return d
@classmethod
def get_from_git(cls, username, store=True):
"""
:param username: the github username
:return: an array of public keys
:rtype: list
"""
uri = 'https://github.com/{:}.keys'.format(username)
content = requests.get(uri).text.strip("\n").split("\n")
d = []
for key in range(0, len(content)):
value = content[key]
thekey = {}
name = "{}_git_{}".format(username, key)
thekey = {
'uri': uri,
'string': value,
'fingerprint': SSHkey._fingerprint(value),
'name': name,
'comment': name,
'cm_id': name,
'source': 'git',
'kind': 'key'
}
thekey["type"], thekey["key"], thekey["comment"] = SSHkey._parse(
value)
if thekey["comment"] is None:
thekey["comment"] = name
d.append(thekey)
if store:
try:
cls.cm.add(thekey)
except:
Console.error("Key already in db", traceflag=False)
if not store:
return d
# noinspection PyProtectedMember,PyUnreachableCode,PyUnusedLocal
@classmethod
def get_from_yaml(cls, filename=None, load_order=None, store=True):
"""
:param filename: name of the yaml file
:return: a SSHKeyManager (dict of keys)
"""
config = None
if filename is None:
# default = Config.path_expand(os.path.join("~", ".cloudmesh", "cloudmesh.yaml"))
# config = ConfigDict("cloudmesh.yaml")
filename = "cloudmesh.yaml"
config = ConfigDict(filename)
elif load_order:
config = ConfigDict(filename, load_order)
else:
Console.error("Wrong arguments")
return
config_keys = config["cloudmesh"]["keys"]
default = config_keys["default"]
keylist = config_keys["keylist"]
uri = Config.path_expand(os.path.join("~", ".cloudmesh", filename))
d = []
for key in list(keylist.keys()):
keyname = key
value = keylist[key]
if os.path.isfile(Config.path_expand(value)):
path = Config.path_expand(value)
if store:
Key.add_from_path(path, keyname)
else:
d.append(Key.add_from_path(path, keyname, store=False))
else:
keytype, string, comment = SSHkey._parse(value)
thekey = {
'uri': 'yaml://{}'.format(uri),
'string': value,
'fingerprint': SSHkey._fingerprint(value),
'name': keyname,
'comment': comment,
'source': 'git',
'kind': 'key'
}
thekey["type"], thekey["key"], thekey[
"comment"] = SSHkey._parse(value)
if thekey["comment"] is None:
thekey["comment"] = keyname
if store:
try:
cls.cm.add(thekey)
except:
Console.error("Key already in db", traceflag=False)
else:
d.append(thekey)
if not store:
return d
"""
take a look into original cloudmesh code, its possible to either specify a key or a filename
the original one is able to figure this out and do the rightthing. We may want to add this
logic to the SSHkey class, so we can initialize either via filename or key string.
It would than figure out the right thing
cloudmesh:
keys:
idrsa: ~/.ssh/id_rsa.pub
cloudmesh:
...
keys:
default: name of the key
keylist:
keyname: ~/.ssh/id_rsa.pub
keyname: ssh rsa hajfhjldahlfjhdlsak ..... comment
github-x: github
"""
@classmethod
def _add_from_sshkey(cls,
sshkey,
keyname=None,
user=None,
source=None,
uri=None):
user = user or cls.cm.user
if keyname is None:
try:
keyname = sshkey['name']
except:
pass
if keyname is None:
print("ERROR: keyname is None")
thekey = {
"kind": "key",
"name": keyname,
"uri": sshkey['uri'],
"source": sshkey['source'],
"fingerprint": sshkey['fingerprint'],
"comment": sshkey['comment'],
"value": sshkey['string'],
"category": "general",
"user": user
}
cls.cm.add(thekey)
@classmethod
def add_key_to_cloud(cls, user, keyname, cloud):
"""
:param user:
:param keyname:
:param cloud:
:param name_on_cloud:
"""
key = cls.cm.find(kind="key", name=keyname, scope="first")
if key is None:
Console.error(
"Key with the name {:} not found in database.".format(keyname))
return
try:
if cloud is not None:
print("Adding key {:} to cloud {:}".format(keyname, cloud))
cloud_provider = CloudProvider(cloud).provider
cloud_provider.add_key_to_cloud(keyname, key["value"])
except Exception as e:
Console.error("problem uploading key {} to cloud {}: {}".format(
keyname, cloud, e.message),
traceflag=False)
@classmethod
def list(cls, category=None, live=False, output="table"):
"this does not work only returns all ceys in the db"
(order, header) = CloudProvider(category).get_attributes("key")
d = cls.cm.find(kind="key", scope="all", output=output)
return Printer.write(d, order=order, header=header, output=output)
@classmethod
def list_on_cloud(cls, cloud, live=False, format="table"):
"""
This method lists all flavors of the cloud
:param cloud: the cloud name
"""
try:
keys = CloudProvider(cloud).provider.list_key(cloud)
for key in keys:
keys[key]["category"] = cloud
if keys is None or keys is []:
return None
(order, header) = CloudProvider(cloud).get_attributes("key")
return Printer.write(keys,
order=order,
header=header,
output=format)
except Exception as ex:
Console.error(ex.message)
@classmethod
def run_command(cls, cmd):
""" Runs a command in a shell, returns the result"""
p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
return p.stdout.read()
@classmethod
def add_azure_key_to_db(cls, key_name, key_path, certificate_path,
pfx_path):
"""
Adds the public key to the existing database model and adds the certificate, key and
fingerprint into the azure key database model.
:param key_name: Key name to be added
:param key_path: Public key path
:param certificate_path: Certificate file path(PEM file)
:param pfx_path: PKCS encoded certificate path
:return:
"""
pprint("add_azure_key_to_db")
# Add to the current DB
cls.add_from_path(key_path,
key_name,
source="ssh",
uri="file://" + key_path)
# Add certificate to the new DB
fingerprint_cmd = "openssl x509 -in " + certificate_path + " -sha1 -noout -fingerprint | sed s/://g"
# print("fingerprint_cmd:", fingerprint_cmd)
fingerprint = cls.run_command(fingerprint_cmd)
fingerprint = fingerprint.split('=')[1]
fingerprint = fingerprint.rstrip('\n')
# pprint("Certificate Fingerprint="+fingerprint)
key_azure_obj = {
"kind": "key_azure",
"name": key_name,
"fingerprint": fingerprint,
"certificate": certificate_path,
"key_path": key_path,
"pfx_path": pfx_path
}
cls.cm.add(key_azure_obj)
Console.info("Azure key added.ok.")
@classmethod
def clear(cls, **kwargs):
raise NotImplementedError()
@classmethod
def refresh(cls, **kwargs):
raise NotImplementedError()
@classmethod
def delete(cls, name=None, cloud=None):
if cloud is not None and name is not None:
result = CloudProvider(cloud).provider.delete_key_from_cloud(name)
elif cloud is not None and name is None:
#
# get a list of keys from cloud
# loop over all keys and use the provider delete from cloud to delete that key
Console.error("delete all keys from cloud not yet implemented")
if name is None:
cls.cm.delete(kind="key", provider="general")
else:
cls.cm.delete(name=name, kind="key", provider="general")
@classmethod
def all(cls, output="dict"):
return cls.cm.find(kind="key", scope="all", output=output)
@classmethod
def find(cls, name=None, output="dict"):
return cls.get(name=name, output=output)
@classmethod
def get(cls, name=None, output="dict"):
"""
Finds the key on the database by name
:param name: name of the key to be found
:return: Query object of the search
"""
if name is None:
return cls.cm.find(kind="key", output=output)
else:
return cls.cm.find(kind="key",
name=name,
output=output,
scope="first")
@classmethod
def set_default(cls, name):
Default.set_key(name)
# deprecated use Default.key
@classmethod
def get_default(cls):
return Default.key
@classmethod
def delete_from_cloud(cls, name, cloud=None):
pass
@classmethod
def _delete_from_db(cls, name=None):
if name is None:
cls.cm.delete(kind='key')
else:
cls.cm.delete(kind='key', name=name)
@classmethod
def select(cls):
options = []
d = cls.get(output='dict')
for i in d:
line = '{}: {}'.format(d[i]['name'], d[i]['fingerprint'])
options.append(line)
num = menu_return_num('KEYS', options)
if num != 'q':
return options[num]
return num
#
# ADD
#
@classmethod
def add_from_path(cls,
path,
keyname=None,
user=None,
source=None,
uri=None,
store=True):
"""
Adds the key to the database based on the path
:param keyname: name of the key or path to the key
:return:
"""
user = user or cls.cm.user
sshkey = SSHkey(Config.path_expand(path))
if store:
cls._add_from_sshkey(sshkey.__key__,
keyname,
user,
source=source,
uri=uri)
else:
return sshkey.__key__
class Group(ListResource):
__kind__ = "group"
__provider__ = "general"
cm = CloudmeshDatabase()
order = ["name", "member", "user", "category", "type", "species"]
# TODO: implement and extend to user
@classmethod
def exists(cls, name, cloud):
"""
checks if the group with the given name exists
"""
raise ValueError("not implemented")
# TODO: implement and extend to user
@classmethod
def check(cls, name, cloud):
"""
checks if the group with the given name exists and raises exception
"""
if not cls.exists(name, cloud):
raise ValueError(
"the default value {} in cloud {} does not exist".format(
name, cloud))
@classmethod
def names(cls):
try:
query = {}
d = cls.cm.find(kind="group", **query)
names = set()
for vm in d:
names.add(vm['name'])
return list(names)
except Exception as ex:
Console.error(ex.message)
@classmethod
def get_vms(cls, name):
"""
returns a list of vms within this group
:param name:
:return:
"""
try:
query = {
"species": "vm",
"scope": "all",
"category": "general",
"kind": "group"
}
if name is not None:
query["name"] = name
d = cls.cm.find(**query)
if d is None:
return None
names = set()
for vm in d:
names.add(vm['member'])
return list(names)
except Exception as ex:
Console.error(ex.message)
@classmethod
def vm_groups(cls, vm):
"""
:param vm: name of the vm
:return: a list of groups the vm is in
"""
try:
query = {"species": "vm", "member": vm}
d = cls.cm.find(kind="group", scope='all', **query)
print("FIND", vm, d)
if d is None:
return None
groups = set()
for vm in d:
groups.add(vm['name'])
return list(groups)
except Exception as ex:
Console.error(ex.message)
@classmethod
def list(cls, name=None, order=None, header=None, output='table'):
"""
lists the default values in the specified format.
TODO: This method has a bug as it uses format and output,
only one should be used.
:param category: the category of the default value. If general is used
it is a special category that is used for global values.
:param format: json, table, yaml, dict, csv
:param order: The order in which the attributes are returned
:param output: The output format.
:return:
"""
if order is None:
order, header = None, None
# order = ['user',
# 'category',
# 'name',
# 'value',
# 'updated_at']
# order, header = Attributes(cls.__kind__, provider=cls.__provider__)
try:
query = {
"provider": cls.__provider__,
"kind": cls.__kind__,
"category": 'general'
}
result = None
if name is not None:
query["name"] = name
result = cls.cm.find(**query)
if result is None:
table = None
else:
table = Printer.write(result, output='table')
return table
except Exception as e:
Console.error("Error creating list", traceflag=False)
Console.error(e.message)
return None
@classmethod
def get_info(cls, category="general", name=None, output="table"):
"""
Method to get info about a group
:param cloud:
:param name:
:param output:
:return:
"""
try:
cloud = category or Default.cloud
args = {"category": category}
if name is not None:
args["name"] = name
group = cls.cm.find(kind="group", output="dict", **args)
return Printer.write(group, order=cls.order, output=output)
except Exception as ex:
Console.error(ex.message)
@classmethod
def add(cls, name=None, species="vm", member=None, category=None):
"""
Add an instance to a new group
or add it to an existing one
:param name:
:param species:
:param member:
:param cloud:
:return:
"""
# user logged into cloudmesh
#user = ConfigDict.getUser(category) or cls.cm.user
user = cls.cm.user
category = category or "general"
try:
# See if group already exists. If yes, add id to the group
data = dotdict({
'member': member,
'name': name,
'kind': 'group',
'provider': 'general'
})
group = cls.cm.find(**data)
if group is None:
t = cls.cm.table(provider="general", kind="group")
group = t(name=name,
member=member,
category="general",
user=user,
species=species)
cls.cm.add(group, replace=False)
return
except Exception as ex:
Console.error(ex.message)
return
@classmethod
def get(cls, **kwargs):
"""
This method queries the database to fetch group(s)
with given name filtered by cloud.
:param name:
:param cloud:
:return:
"""
query = dict(kwargs)
if 'output' in kwargs:
for key, value in kwargs.items():
if value is None:
query[key] = "None"
del query['output']
try:
print("QQQ"), query
group = cls.cm.find(kind="group", **query)
print("gggg", group)
if group is not None \
and "output" in kwargs:
d = {"0": group}
group = Printer.write(d)
return group
except Exception as ex:
Console.error(ex.message)
@classmethod
def delete(cls, name=None):
"""
Method to delete a group from
the cloudmesh database
:param name:
:param cloud:
:return:
"""
try:
# group = cls.get(name=name, category=category)
args = {}
if name is not None:
args["name"] = name
group = cls.cm.find(provider='general',
kind="group",
scope='all',
output="dict",
**args)
if group:
# Delete VM from cloud before deleting group
for vm in group:
server = vm["member"]
groups = Group.vm_groups(server)
if groups is not None and len(groups) == 1:
try:
Vm.delete(name=server, servers=[server])
except Exception as e:
Console.error(
"Failed to delete VM {}, error: {}".format(
vm, e),
traceflag=False)
Console.error(e.message)
continue
# Delete group record in local db
for element in group:
cls.cm.delete(**element)
cls.cm.save()
return "Delete. ok."
else:
return None
except Exception as ex:
Console.error(ex.message)
@classmethod
def remove(cls, name, member):
"""
Method to remove an ID from the group
in the cloudmesh database
:param name:
:param id:
:param category:
:return:
"""
try:
# group = cls.get(name=name, category=category)
args = {
"name": name,
"category": "general",
"member": member,
}
# Find an existing group with name & category
group = cls.cm.find(kind="group",
scope='all',
output="dict",
**args)
print("YYYY", group, args)
if group is not None:
for element in group:
print("ELEMENT", element)
cls.cm.delete(**element)
return "Removed {} from the group {}. ok.".format(member, name)
except Exception as ex:
Console.error(ex.message)
return None
@classmethod
def copy(cls, _fromName, _toName):
"""
Method to make copy of a group
:param _fromName:
:param _toName:
:return:
"""
try:
from_args = {"name": _fromName}
to_args = {"name": _toName}
_fromGroup = cls.cm.find(kind="group",
scope="all",
output="dict",
**from_args)
_toGroup = cls.cm.find(kind="group",
scope="all",
output="dict",
**to_args)
# print ("A")
# pprint (_fromGroup)
# print ("B")
# pprint(_toGroup)
if _fromGroup is not None:
for from_element in _fromGroup:
member = from_element["member"]
species = from_element["species"]
category = from_element["category"]
print("TTT", _toName)
cls.add(name=_toName,
species=species,
member=member,
category=category)
cls.cm.save()
Console.ok("Copy from group {} to group {}. ok.".format(
_fromName, _toName))
else:
Console.error(
"Group [{}] does not exist in the cloudmesh database!".
format(_fromName),
traceflag=False)
return None
except Exception as ex:
Console.error(ex.message)
@classmethod
def merge(cls, group_a, group_b, merged_group):
"""
Method to merge two groups into
one group
:param group_a:
:param group_b:
:param merged_group:
:return:
"""
cls.copy(group_a, merged_group)
cls.copy(group_b, merged_group)
# TODO: this is dependent on the provider This needs to be imported from the provider
@classmethod
def to_dict(cls, item):
"""
Method to convert input to a dict
:param item:
:return:
"""
d = {item.id: {}}
for key in list(item.__dict__.keys()):
if not key.startswith("_sa"):
d[item.id][key] = str(item.__dict__[key])
return d
class Cluster(CLUSTER): # list abstraction see other commands
cm = CloudmeshDatabase()
def __init__(self, *args, **kwargs):
# Use the table defined in the model, but we need to look up
# the provider object dynamically
super(Cluster, self).__init__(*args, **kwargs)
self.provider = CloudProvider(self.cloud).provider.cloud_type
# put this cluster in the database, the 'name' attribute must
# be unique
try:
self.cm.insert(self)
except IntegrityError as e:
line = 'UNIQUE constraint failed: {}.name'\
.format(self.__tablename__)
if line in e.message:
raise ClusterNameClashException(self.__tablename__, self.name)
@classmethod
def from_name(cls, name):
return cls.cm.select(Cluster, name=name).one()
def __iter__(self):
return iter(self.list())
def list(self):
"""List the nodes in the cluster.
The type of the instance is determined by the provider.
:returns: the nodes of the cluster
:rtype: :class:`list` of instances
"""
table = self.cm.vm_table_from_provider(self.provider)
return self.cm.select(table, cluster=self.name).all()
def delete(self, force=False):
"""Delete this cluster and all component nodes"""
for node in self:
Vm.delete(servers=[node.name], force=force)
self.cm.delete_(self.__class__, cm_id=self.cm_id)
def create(self, sleeptime_s=5):
"""Boot all nodes in this cluster
:param float sleeptime_s: sleep this number of seconds between
polling VMs for ACTIVE status
"""
for _ in xrange(self.count - len(self.list())):
self.add()
def add(self):
"""Boots a new instance and adds it to this cluster"""
provider = Provider.from_cloud(self.cloud)
Console.info('Booting VM for cluster {}'.format(self.name))
node = provider.boot(key=self.key,
image=self.image,
flavor=self.flavor,
secgroup=self.secgroup,
cluster=self.name,
username=self.username)
if self.assignFloatingIP:
node.create_ip()
def remove(self, cm_id):
"""Removes a node to the cluster, but otherwise leaves it intact.
See :meth:`delete` to delete this cluster
:param int cm_id: the node id of the instance to remove
"""
table = self.cm.vm_table_from_provider(self.provider)
self.cm.update_(table,
where={'cm_id': cm_id},
values={'cluster': None})
def modify(self):
"Modifies the cluster"
raise NotImplementedError()
def terminate(self):
"Terminates the cluster"
raise NotImplementedError()
def suspend(self):
"Suspends the cluster"
raise NotImplementedError()
def resume(self):
"Resumes the cluster"
raise NotImplementedError()
def add_key(self, public_key):
"Adds an ssh public key to the cluster"
raise NotImplementedError()
def remove_key(self, public_key):
"Removes an ssh public key from the cluster"
raise NotImplementedError()
def enable_cross_ssh_login(self):
"Enables each node to log into all other nodes of the cluster"
raise NotImplementedError()
def disable_cross_ssh_login(self):
raise NotImplementedError()
def delete_key(self):
raise NotImplementedError()
class Workflow(ListResource):
cm = CloudmeshDatabase()
@classmethod
def refresh(cls, cloud):
"""
This method would refresh the workflow list by first clearing
the database, then inserting new data
:param cloud: the cloud name
"""
Console.TODO(
"this method is not yet implemented. dont implement this yet")
return
return cls.cm.refresh('workflow', cloud)
@classmethod
def delete(cls, cloud, id):
print(id)
cls.cm.delete(kind="workflow", category='general', cm_id=id)
return True
@classmethod
def list(cls, name, live=False, format="table"):
"""
This method lists all workflows of the cloud
:param cloud: the cloud name
"""
# Console.TODO("this method is not yet implemented")
# return
try:
elements = cls.cm.find(kind="workflow", category='general')
# pprint(elements)
# (order, header) = CloudProvider(cloud).get_attributes("workflow")
order = None
header = None
# Console.msg(elements)
return Printer.write(elements,
order=order,
header=header,
output=format)
except Exception as ex:
Console.error(ex.message)
@classmethod
def details(cls, cloud, id, live=False, format="table"):
elements = cls.cm.find(kind="workflow", category='general', cm_id=id)
Console.msg(elements)
order = None
header = None
# Console.TODO("this method is not yet implemented")
return Printer.write(elements,
order=order,
header=header,
output=format)
@classmethod
def save(cls, cloud, name, str):
workflow = {
"category": "general",
"kind": "workflow",
"name": name,
"workflow_str": str
}
cls.cm.add(workflow, replace=False)
cls.cm.save()
return "Workflow saved in database!"
@classmethod
def run(cls, cloud, id):
elements = cls.cm.find(kind="workflow", category='general', cm_id=id)
Console.msg(elements)
order = None
Console.msg("Executing")
header = None
return elements
class Cluster(CLUSTER): # list abstraction see other commands
cm = CloudmeshDatabase()
def __init__(self, *args, **kwargs):
# Use the table defined in the model, but we need to look up
# the provider object dynamically
kwargs['cloud'] = kwargs.get('cloud', Default.cloud)
kwargs['image'] = kwargs.get('image', Default.image)
kwargs['username'] = kwargs.get('username', Image.guess_username(kwargs['image']))
kwargs['flavor'] = kwargs.get('flavor', Default.flavor)
kwargs['key'] = kwargs.get('key', Default.key)
kwargs['secgroup'] = kwargs.get('secgroup', Default.secgroup)
super(Cluster, self).__init__(*args, **kwargs)
self.provider = CloudProvider(self.cloud).provider.cloud_type
# put this cluster in the database, the 'name' attribute must
# be unique
try:
self.cm.insert(self)
except IntegrityError as e:
line = 'UNIQUE constraint failed: {}.name'\
.format(self.__tablename__)
if line in e.message:
raise ClusterNameClashException(self.__tablename__,
self.name)
@classmethod
def from_name(cls, name):
return cls.cm.select(Cluster, name=name).one()
def __iter__(self):
return iter(self.list())
def list(self):
"""List the nodes in the cluster.
The type of the instance is determined by the provider.
:returns: the nodes of the cluster
:rtype: :class:`list` of instances
"""
table = self.cm.vm_table_from_provider(self.provider)
return self.cm.select(table, cluster=self.name).all()
def delete(self, force=False):
"""Delete this cluster and all component nodes"""
for node in self:
Vm.delete(servers=[node.name], force=force)
self.cm.delete(kind="vm", provider=self.provider, name=node.name)
self.cm.delete_(self.__class__, cm_id=self.cm_id)
def create(self, sleeptime_s=5):
"""Boot all nodes in this cluster
:param float sleeptime_s: sleep this number of seconds between
polling VMs for ACTIVE status
"""
for _ in xrange(self.count - len(self.list())):
self.add()
def add(self):
"""Boots a new instance and adds it to this cluster"""
provider = Provider.from_cloud(self.cloud)
Console.info('Booting VM for cluster {}'.format(self.name))
node = provider.boot(
key = self.key,
image = self.image,
flavor = self.flavor,
secgroup = self.secgroup,
cluster = self.name,
username = self.username
)
if self.assignFloatingIP:
node.create_ip()
def remove(self, cm_id):
"""Removes a node to the cluster, but otherwise leaves it intact.
See :meth:`delete` to delete this cluster
:param int cm_id: the node id of the instance to remove
"""
table = self.cm.vm_table_from_provider(self.provider)
self.cm.update_(
table,
where={'cm_id': cm_id},
values={'cluster': None}
)
def modify(self):
"Modifies the cluster"
raise NotImplementedError()
def terminate(self):
"Terminates the cluster"
raise NotImplementedError()
def suspend(self):
"Suspends the cluster"
raise NotImplementedError()
def resume(self):
"Resumes the cluster"
raise NotImplementedError()
def add_key(self, public_key):
"Adds an ssh public key to the cluster"
raise NotImplementedError()
def remove_key(self, public_key):
"Removes an ssh public key from the cluster"
raise NotImplementedError()
def enable_cross_ssh_login(self, useFloating=True, keytype='rsa', bits=4096, comment='CM Cluster Cross-SSH'):
"Enables each node to log into all other nodes of the cluster"
ssh = [
'ssh',
'-o', 'UserKnownHostsFile=/dev/null',
'-o', 'StrictHostKeyChecking=no',
'-l', self.username,
]
ssh_keygen = [
'ssh-keygen',
'-f', '.ssh/id_{}'.format(keytype),
'-b', str(bits),
'-t', keytype,
'-C', quote(comment),
'-N', quote(''),
]
slurp = [
'scp',
'-o', 'UserKnownHostsFile=/dev/null',
'-o', 'StrictHostKeyChecking=no',
]
with tempdir() as workdir:
def auth_keys_f(node):
return os.path.join(workdir, node.name, 'authorized_keys')
def pubkey_f(node):
return os.path.join(workdir, node.name, 'id_{}.pub'.format(keytype))
def ip_f(node):
return node.floating_ip if useFloating else node.static_ip
for node in self:
outdir = os.path.join(workdir, node.name)
os.makedirs(outdir)
ip = ip_f(node)
# cleanup
rm = ssh + [ip] + [
'rm',
'-f',
'.ssh/id_{}'.format(keytype),
'.ssh/id_{}.pub'.format(keytype),
]
Subprocess(rm)
genkey = ssh + [ip] + ssh_keygen
Subprocess(genkey)
pubkey = '{}@{}:.ssh/id_{}.pub'.format(node.username, ip, keytype)
auth_keys = '{}@{}:.ssh/authorized_keys'.format(node.username, ip)
scp = slurp + [pubkey] + [auth_keys] + [outdir]
Subprocess(scp)
for nodeA in self:
auth = AuthorizedKeys.from_authorized_keys(auth_keys_f(nodeA))
# add the keys for all the other machines
for nodeB in self:
with open(pubkey_f(nodeB)) as fd:
for line in itertools.imap(str.strip, fd):
if not line:
continue
auth.add(line)
# save new authorized_keys
with open(auth_keys_f(nodeA), 'w') as fd:
fd.write(auth.text())
for node in self:
path = auth_keys_f(node)
remote = '{}@{}:.ssh/authorized_keys'.format(node.username, ip_f(node))
scp = slurp + [path] + [remote]
Subprocess(scp)
def disable_cross_ssh_login(self):
raise NotImplementedError()
def delete_key(self):
raise NotImplementedError()
class Var(object):
"""
Cloudmesh contains the concept of defaults. Defaults can have
categories (we will rename cloud to categories). A category can be a
cloud name or the name 'general'. The category general is a 'global'
name space and contains defaults of global value (in future we will
rename the value to global).
"""
__kind__ = "var"
__provider__ = "general"
cm = CloudmeshDatabase()
"""cm is a static variable so that db is used uniformly."""
@classmethod
def list(cls, order=None, header=None, output='table'):
"""
lists the default values in the specified format.
TODO: This method has a bug as it uses format and output,
only one should be used.
:param category: the category of the default value. If general is used
it is a special category that is used for global values.
:param format: json, table, yaml, dict, csv
:param order: The order in which the attributes are returned
:param output: The output format.
:return:
"""
if order is None:
order, header = None, None
# order, header = Attributes(cls.__kind__, provider=cls.__provider__)
try:
result = cls.cm.all(provider=cls.__provider__, kind=cls.__kind__)
return (Printer.write(result, order=order, output=output))
except Exception as e:
Console.error("Error creating list", traceflag=False)
Console.error(e.message)
return None
#
# GENERAL SETTER AND GETTER METHOD
#
@classmethod
def set(cls, key, value, user=None, type='str'):
"""
sets the default value for a given category
:param key: the dictionary key of the value to store it at.
:param value: the value
:param user: the username to store this default value at.
:return:
"""
try:
o = cls.get(name=key)
if o is not None:
cls.cm.update(kind=cls.__kind__,
provider=cls.__provider__,
filter={'name': key},
update={
'value': value,
'type': type
})
else:
t = cls.cm.table(provider=cls.__provider__, kind=cls.__kind__)
o = t(name=key, value=value, type=type)
cls.cm.add(o)
cls.cm.save()
except Exception as e:
Console.error("problem setting key value {}={}".format(key, value),
traceflag=False)
Console.error(e.message)
@classmethod
def get(cls, name=None, output='dict', scope='first'):
"""
returns the value of the first objects matching the key
with the given category.
:param key: The dictionary key
:param category: The category
:return:
"""
o = cls.cm.find(kind=cls.__kind__,
provider=cls.__provider__,
output=output,
scope=scope,
name=name)
return o
@classmethod
def delete(cls, name):
cls.cm.delete(name=name, provider=cls.__provider__, kind=cls.__kind__)
@classmethod
def clear(cls):
"""
deletes all default values in the database.
:return:
"""
cls.cm.delete(provider=cls.__provider__, kind=cls.__kind__)
class List(object):
cm = CloudmeshDatabase()
@classmethod
def list(cls,
kind,
cloud,
user=None,
tenant=None,
order=None,
header=None,
output="table"):
"""
Method lists the data in the db for
given cloud and of given kind
:param kind:
:param cloud:
:param tenant:
:param user:
:param order:
:param header:
:param output:
:return:
"""
try:
# get the model object
table = cls.cm.get_table(kind)
filter = {}
if cloud is not None:
filter["category"] = cloud
if user is not None:
filter["user"] = user
if tenant is not None:
filter["tenant"] = tenant
elements = cls.cm.find(table, **filter)
if elements is not None or elements is not {}:
# convert the output to a dict
return (Printer.write(elements,
order=order,
header=header,
output=output))
else:
return None
except Exception as ex:
Console.error(ex.message)
#
# TODO: don't we have not already a conversion method
#
@classmethod
def toDict(cls, item):
"""
Method converts the item to a dict
:param item:
:return:
"""
# Convert to dict & print table
d = {}
# If list, iterate to form dict
if isinstance(item, list):
for element in item:
d[element.id] = {}
for key in list(element.__dict__.keys()):
if not key.startswith("_sa"):
d[element.id][key] = str(element.__dict__[key])
# Form dict without iterating
else:
d[item.id] = {}
for key in list(item.__dict__.keys()):
if not key.startswith("_sa"):
d[item.id][key] = str(item.__dict__[key])
# return the dict
return d
class BatchProviderSLURM(BatchProviderBase):
cm = CloudmeshDatabase()
kind = "slurm"
@classmethod
def queue(cls, cluster, format='json', job=None):
try:
args = 'squeue '
if job is not None:
if job.isdigit():
args += ' -j {} '.format(str(job)) # search by job id
else:
args += ' -n {} '.format(job) # search by job name
f = '--format=%all'
args += f
result = Shell.ssh(cluster, args)
# TODO: process till header is found...(Need a better way)
l = result.splitlines()
for i, res in enumerate(l):
if 'ACCOUNT|GRES|' in res:
result = "\n".join(str(x) for x in l[i:])
break
parser = TableParser(strip=True)
d = parser.to_dict(result)
# add cluster and updated to each entry
for key in list(d.keys()):
d[key]['cluster'] = cluster
d[key]['updated'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if format == 'json':
return json.dumps(d, indent=4, separators=(',', ': '))
else:
return (Printer.write(d,
order=['cluster',
'jobid',
'partition',
'name',
'user',
'st',
'time',
'nodes',
'nodelist',
'updated'],
output=format))
except Exception as e:
Error.traceback(e)
return e
@classmethod
def info(cls, cluster, format='json', all=False):
if all:
result = Shell.ssh(cluster, 'sinfo --format=\"%all\"')
else:
result = Shell.ssh(
cluster,
'sinfo --format=\"%P|%a|%l|%D|%t|%N\"')
# ignore leading lines till header is found
l = result.splitlines()
for i, res in enumerate(l):
if 'PARTITION|AVAIL|' in res:
result = "\n".join(l[i:])
break
parser = TableParser(strip=False)
d = parser.to_dict(result)
# add cluster and updated to each entry
for key in list(d.keys()):
d[key]['cluster'] = cluster
d[key]['updated'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if format == 'json':
return json.dumps(d, indent=4, separators=(',', ': '))
else:
return (Printer.write(d,
order=['cluster',
'partition',
'avail',
'timelimit',
'nodes',
'state',
'nodelist',
'updated'],
output=format))
@classmethod
def test(cls, cluster, time):
result = Shell.ssh(cluster,
"srun -n1 -t {} echo '#CLOUDMESH: Test ok'".format(
time))
return result
@classmethod
def run(cls, cluster, group, cmd, **kwargs):
# determine the script name..
#
# TODO: script count is variable in data base, we test if fil exists and if it
# does increase counter till we find one that does not, that will be new counter.
# new counter will than be placed in db.
#
# define get_script_name(directory, prefix, counter)
# there maybe s a similar thing already in the old cloudmesh
#
# if not kwargs['-name']:
#
# old_count = Shell.ssh(cluster,
# "ls {}*.sh | wc -l | sed 's/$/ count/'".
# format(username))
# c = [f for f in old_count.splitlines() if 'count' in f]
# script_count = c[0].split()[0]
# else:
# script_count = kwargs['-name']
config = cls.read_config(cluster)
if config["credentials"]["username"] == 'TBD':
return "Please enter username in cloudmesh.yaml for cluster {}".format(cluster)
cls.incr()
data = {
"cluster": cluster,
"count": cls.counter(),
"username": config["credentials"]["username"],
"remote_experiment_dir": config["default"]["experiment_dir"],
"queue": config["default"]["queue"],
"id": None,
"nodes": 1,
"tasks_per_node": 1,
}
data["script_base_name"] = "{username}-{count}".format(**data)
data["script_name"] = "{username}-{count}.sh".format(**data)
data["script_output"] = "{username}-{count}.out".format(**data)
data["script_error"] = "{username}-{count}.err".format(**data)
data["remote_experiment_dir"] = \
"{remote_experiment_dir}/{count}".format(**data).format(**data)
data["group"] = group
# overwrite defaults
option_mapping = {'-t': '{tasks_per_node}'.format(**data),
'-N': '{nodes}'.format(**data),
'-p': '{queue}'.format(**data),
'-o': '{script_output}'.format(**data),
'-D': '{remote_experiment_dir}'.format(**data),
'-e': '{script_error}'.format(**data)}
# map(lambda k, v:
# option_mapping.__setitem__(k, kwargs.get(k) or v),
# option_mapping.items())
#
# rewrite for better readability
for (k, v) in iteritems(option_mapping):
option_mapping[k] = kwargs.get(k) or v
config = cls.read_config(cluster)
project = None
try:
project = config["credentials"]["project"]
if project.lower() not in ["tbd", "none"]:
option_mapping["-A"] = project
except:
pass
for key in option_mapping:
data[key] = option_mapping[key]
# create the options for the script
options = ""
for key, value in option_mapping.items():
options += '#SBATCH {} {}\n'.format(key, value)
cls.create_remote_dir(cluster, data["remote_experiment_dir"])
# if the command is a script, copy the script
if os.path.isfile(Config.path_expand(cmd)):
_from = Config.path_expand(cmd)
_to = '{cluster}:{remote_experiment_dir}'.format(**data)
local_file_name = cmd.split('/')[-1]
Shell.execute("rsync", [_from, _to])
data["command"] = '{remote_experiment_dir}/{local_file_name}'.format(local_file_name=local_file_name,
**data)
else:
data["command"] = cmd
data["options"] = options
script = textwrap.dedent(
"""
#! /bin/sh
{options}
echo '#CLOUDMESH: BATCH ENVIRONMENT'
echo 'BASIL_RESERVATION_ID:' $BASIL_RESERVATION_ID
echo 'SLURM_CPU_BIND:' $SLURM_CPU_BIND
echo 'SLURM_JOB_ID:' $SLURM_JOB_ID
echo 'SLURM_JOB_CPUS_PER_NODE:' $SLURM_JOB_CPUS_PER_NODE
echo 'SLURM_JOB_DEPENDENCY:' $SLURM_JOB_DEPENDENCY
echo 'SLURM_JOB_NAME:' $SLURM_JOB_NAME
echo 'SLURM_JOB_NODELIST:' $SLURM_JOB_NODELIST
echo 'SLURM_JOB_NUM_NODES:' $SLURM_JOB_NUM_NODES
echo 'SLURM_MEM_BIND:' $SLURM_MEM_BIND
echo 'SLURM_TASKS_PER_NODE:' $SLURM_TASKS_PER_NODE
echo 'MPIRUN_NOALLOCATE:' $MPIRUN_NOALLOCATE
echo 'MPIRUN_NOFREE:' $MPIRUN_NOFREE
echo 'SLURM_NTASKS_PER_CORE:' $SLURM_NTASKS_PER_CORE
echo 'SLURM_NTASKS_PER_NODE:' $SLURM_NTASKS_PER_NODE
echo 'SLURM_NTASKS_PER_SOCKET:' $SLURM_NTASKS_PER_SOCKET
echo 'SLURM_RESTART_COUNT:' $SLURM_RESTART_COUNT
echo 'SLURM_SUBMIT_DIR:' $SLURM_SUBMIT_DIR
echo 'MPIRUN_PARTITION:' $MPIRUN_PARTITION
d=$(date)
echo \"#CLOUDMESH: status, start, $d\"
srun -l echo \"#CLOUDMESH: status, start, $d\"
srun -l {command}
d=$(date)
srun -l echo \"#CLOUDMESH: status, finished, $d\"
d=$(date)
echo \"#CLOUDMESH: status, finished, $d\"
"""
).format(**data).replace("\r\n", "\n").strip()
_from = Config.path_expand('~/.cloudmesh/{script_name}'.format(**data))
_to = '{cluster}:{remote_experiment_dir}'.format(**data)
data["from"] = _from
data["to"] = _to
data["script"] = script
# write the script to local
# print(_from)
# print(_to)
with open(_from, 'w') as local_file:
local_file.write(script)
# copy to remote host
Shell.scp(_from, _to)
# delete local file
# Shell.execute('rm', _from)
# import sys; sys.exit()
# run the sbatch command
cmd = 'sbatch {remote_experiment_dir}/{script_name}'.format(**data)
data["cmd"] = cmd
# print ("CMD>", cmd)
result = Shell.ssh(cluster, cmd)
data["output"] = result
# find id
for line in result.split("\n"):
# print ("LLL>", line)
if "Submitted batch job" in line:
data["job_id"] = int(line.replace("Submitted batch job ", "").strip())
break
#
# HACK, should not depend on Model.py
#
# from cloudmesh_client.db.model import BATCHJOB
# name = ""
# BATCHJOB(name,
# cluster=data["cluster"],
# id=data["id"],
# script=data["script"]) # has user and username which seems wrong
# here what we have in data and want to store the - options are obviously wrong
# and need to be full names
# noinspection PyPep8,PyPep8
"""
{'-D': '/N/u/gvonlasz/experiment/3',
'-N': '1',
'-o': 'gvonlasz-3.out',
'-p': 'delta',
'-t': '1',
'cluster': 'india',
'cmd': 'sbatch /N/u/gvonlasz/experiment/3/gvonlasz-3.sh',
'command': 'uname',
'count': 3,
'from': '/Users/big/.cloudmesh/gvonlasz-3.sh',
'id': 1346,
'options': '#SBATCH -t 1\n#SBATCH -o gvonlasz-3.out\n#SBATCH -N 1\n#SBATCH -p delta\n#SBATCH -D /N/u/gvonlasz/experiment/3\n',
'output': 'Submitted batch job 1346',
'queue': 'delta',
'remote_experiment_dir': '/N/u/gvonlasz/experiment/3',
'script': "#! /bin/sh\n#SBATCH -t 1\n#SBATCH -o gvonlasz-3.out\n#SBATCH -N 1\n#SBATCH -p delta\n#SBATCH -D /N/u/gvonlasz/experiment/3\n\nsrun -l echo '#CLOUDMESH: Starting'\nsrun -l uname\nsrun -l echo '#CLOUDMESH: Test ok'",
'script_base_name': 'gvonlasz-3',
'script_name': 'gvonlasz-3.sh',
'script_output': 'gvonlasz-3.out',
'to': 'india:/N/u/gvonlasz/experiment/3',
'username': '******'}
"""
"""
we also want to store what part of the .out file,
BASIL_RESERVATION_ID:
SLURM_CPU_BIND:
SLURM_JOB_ID: 1351
SLURM_JOB_CPUS_PER_NODE: 12
SLURM_JOB_DEPENDENCY:
SLURM_JOB_NAME: gvonlasz-8.sh
SLURM_JOB_NODELIST: d001
SLURM_JOB_NUM_NODES: 1
SLURM_MEM_BIND:
SLURM_TASKS_PER_NODE: 12
MPIRUN_NOALLOCATE:
MPIRUN_NOFREE:
SLURM_NTASKS_PER_CORE:
SLURM_NTASKS_PER_NODE:
SLURM_NTASKS_PER_SOCKET:
SLURM_RESTART_COUNT:
SLURM_SUBMIT_DIR: /N/u/gvonlasz
MPIRUN_PARTITION:
so maybe we want to use some of the names here as they reflect the env vars
"""
#
# add data to database
#
# remove the - options
for key in ['-t', '-N', '-p', '-o', '-D', '-e']:
if key in data:
print(key, data[key])
del data[key]
data['status'] = 'started'
cls.add_db(**data)
return data
@classmethod
def delete(cls, cluster, job, group=None):
"""
This method is used to terminate a job with the specified or a group of jobs
job_id or job_name in a given cluster
:param group:
:param cluster: the cluster like comet
:param job: the job id or name
:return: success message or error
"""
try:
if group is not None:
# get the job ids from the db
arguments = {'cluster': cluster,
'group': group}
db_jobs = cls.cm.find('batchjob',
**arguments)
list1 = []
for i in db_jobs:
list1.append(db_jobs[i]['job_id'])
# read active jobs
active_jobs = json.loads(cls.queue(cluster))
list2 = []
for i in active_jobs:
list2.append(active_jobs[i]['jobid'])
# find intersection
res = set(list1).intersection(set(list2))
if res is not None:
for j in res:
cmd = 'scancel {}'.format(str(j))
Shell.ssh(cluster, cmd)
print("Deleted {}".format(j))
return "All jobs for group {} killed successfully".format(group)
else:
args = 'scancel '
if job.isdigit():
args += job
else:
args += "-n {}".format(job)
Shell.ssh(cluster, args)
return "Job {} killed successfully".format(job)
except Exception as ex:
print("in exceptio")
print(ex)
return ex
@classmethod
def add_db(cls, **kwargs):
kwargs['name'] = kwargs.get('script_name')
db_obj = {0: {"batchjob": kwargs}}
cls.cm.add_obj(db_obj)
cls.cm.save()
class SecGroup(ListResource):
cm = CloudmeshDatabase()
"""
NOT USED
@classmethod
def convert_list_to_dict(cls, os_result):
d = {}
for i, obj in enumerate(os_result):
d[i] = {}
d[i]["Id"] = obj.id
d[i]["Name"] = obj.name
d[i]["Description"] = obj.description
return d
"""
# noinspection PyPep8
@classmethod
def convert_rules_to_dict(cls, os_result):
d = {}
for i, obj in enumerate(os_result):
if obj["ip_range"]["cidr"]:
ip_range = obj["ip_range"]["cidr"]
else:
ip_range = "0.0.0.0/0"
d[i] = {
"IP Protocol": obj["ip_protocol"],
"From Port": obj["from_port"],
"To Port": obj["to_port"],
"IP Range": ip_range
}
return d
@classmethod
def refresh(cls, cloud):
"""
This method would refresh the secgroup list by first clearing
the database, then inserting new data
:param cloud: the cloud name
"""
return cls.cm.refresh('secgroup', cloud)
@classmethod
def add_rule_to_db(cls, group=None, name=None, from_port=None, to_port=None, protocol=None, cidr=None):
old_rule = {
"category": "general",
"kind": "secgrouprule",
"name": name,
"group": group
}
cls.cm.delete(**old_rule)
try:
rule = {
"category": "general",
"kind": "secgrouprule",
"group": group,
"name": name,
'protocol': protocol,
'fromPort': from_port,
'toPort': to_port,
'cidr': cidr
}
cls.cm.add(rule, replace=False)
except Exception as ex:
Console.error("Problem adding rule")
@classmethod
def upload(cls, cloud=None, group=None):
if cloud is None:
clouds = ConfigDict("cloudmesh.yaml")["cloudmesh"]["active"]
else:
clouds = [cloud]
if group is None:
rules = cls.list(output='dict')
groups = set()
for g in rules:
r = rules[g]
groups.add(r["group"])
groups = list(groups)
else:
groups = [group]
for cloud in clouds:
for g in groups:
cls.delete_all_rules_cloud(cloud, g)
group = cls.get(name=g, cloud=cloud)
group_cloud = cls.get_group_cloud(cloud, g)
if not group_cloud:
cls.add_group_cloud(cloud, g)
rules = cls.list_rules(group=g, output="dict")
if rules:
for ruleid in rules:
rule = rules[ruleid]
rulename = rule["name"]
cls.add_rule_cloud(cloud, g, rulename)
'''
SecGroup.delete(category=c, group=g)
uuid = SecGroup.create(category=c, group=g)
for key in rules:
r = rules[key]
if r["group"] == g:
SecGroup.add_rule(c,uuid,r["fromPort"],r["toPort"] , r['protocol'],r['cidr'])
# create group
'''
@classmethod
def create(cls, group=None, category=None):
"""
Method creates a new security group in database
& returns the uuid of the created group
:param group:
:param category:
:return:
"""
# Create the security group in given cloud
try:
cloud_provider = CloudProvider(category).provider
secgroup = cloud_provider.create_secgroup(group)
if secgroup:
uuid = secgroup.id
return uuid
else:
print("Failed to create security group, {}".format(secgroup))
except Exception as e:
print(
"Exception creating security group in cloud, {}".format(e))
return None
@classmethod
def list(cls,
group=None,
name=None,
category='general',
output='table',
scope='all'):
"""
This method queries the database to fetch list of secgroups
filtered by cloud.
:param cloud:
:return:
"""
query = dotdict({
"kind": "secgrouprule",
"scope": "all"
})
if category is "general":
if group is not None:
query.group = group
if name is not None:
query.name = name
query.category = category
elements = cls.cm.find(**query)
else:
elements = CloudProvider(category).provider.list_secgroup_rules(category)
if elements is None:
return None
else:
# pprint(elements)
#
# BUG this should not depend on cloud, but on "general"
#
# (order, header) = CloudProvider(cloud).get_attributes("secgroup")
order = ['name', 'group', 'fromPort', 'toPort', 'cidr', 'protocol']
header = None
return Printer.write(elements,
order=order,
header=header,
output=output)
@classmethod
def list_rules(cls, group=None, output='table'):
"""
This method gets the security group rules
from the cloudmesh database
:param uuid:
:return:
"""
try:
if group is None:
rules = cls.cm.find(kind="secgrouprule")
else:
args = {
"group": group
}
rules = cls.cm.find(kind="secgrouprule", **args)
# check if rules exist
if rules is None:
return "No rules for security group={} in the database. Try cm secgroup refresh.".format(group)
# return table
return (Printer.write(rules,
order=["user",
"group",
"category",
"name",
"fromPort",
"toPort",
"protocol",
"cidr"],
output=output))
except Exception as ex:
Console.error("Listing Security group rules")
return None
@classmethod
def enable_ssh(cls, secgroup_name='default', cloud="general"):
ret = False
if cloud in LibcloudDict.Libcloud_category_list:
Console.info("Creating and adding security group for libcloud")
cloud_provider = CloudProvider(cloud).provider
cloud_provider.create_sec_group(cloud, secgroup_name)
cloud_provider.enable_ssh(cloud, secgroup_name)
else:
cloud_provider = CloudProvider(cloud).provider.provider
secgroups = cloud_provider.security_groups.list()
for asecgroup in secgroups:
if asecgroup.name == secgroup_name:
rules = asecgroup.rules
rule_exists = False
# structure of a secgroup rule:
# {u'from_port': 22, u'group': {}, u'ip_protocol': u'tcp', u'to_port': 22, u'parent_group_id': u'UUIDHERE', u'ip_range': {u'cidr': u'0.0.0.0/0'}, u'id': u'UUIDHERE'}
for arule in rules:
if arule["from_port"] == 22 and \
arule["to_port"] == 22 and \
arule["ip_protocol"] == 'tcp' and \
arule["ip_range"] == {'cidr': '0.0.0.0/0'}:
# print (arule["id"])
rule_exists = True
break
if not rule_exists:
cloud_provider.security_group_rules.create(
asecgroup.id,
ip_protocol='tcp',
from_port=22,
to_port=22,
cidr='0.0.0.0/0')
# else:
# print ("The rule allowing ssh login did exist!")
ret = True
break
# print ("*" * 80)
# d = SecGroup.convert_list_to_dict(secgroups)
# print (d)
return ret
@classmethod
def get(cls, name=None, cloud="general"):
"""
This method queries the database to fetch secgroup
with given name filtered by cloud.
:param name:
:param cloud:
:return:
"""
try:
args = {
"name": name,
'scope': 'fisrt',
'kind': "secgroup",
"output": "object",
}
if cloud is not None and cloud is not 'general':
args["category"] = cloud
secgroup = cls.cm.find(**args)
if secgroup is None:
return None
else:
return secgroup[0]
except Exception as ex:
Console.error("get secgroup")
return None
@classmethod
def add_rule(cls, cloud, secgroup_uuid, from_port, to_port, protocol, cidr):
try:
# Get the nova client object
cloud_provider = CloudProvider(cloud).provider
# Create add secgroup rules to the cloud
args = {
'uuid': secgroup_uuid,
'protocol': protocol,
'from_port': from_port,
'to_port': to_port,
'cidr': cidr
}
rule_id = cloud_provider.add_secgroup_rule(**args)
# create local db record
rule = {"kind": "secgrouprule",
"uuid": str(rule_id),
"category": cloud,
"fromPort": from_port,
"toPort": to_port,
"protocol": protocol,
"cidr": cidr}
"""
cls.cm.add(**rule)
cls.cm.save()
"""
Console.ok("Added rule {category} {uuid} {fromPort} {toPort} {protocol} {cidr}"
.format(**rule))
except Exception as ex:
if "This rule already exists" in ex.message:
Console.ok("Rule already exists. Added rule.")
return
else:
Console.error(ex.message, ex)
return
@classmethod
def reset_defaults(cls):
secgroup = "{}-default".format(Default.user)
Default.set_secgroup(secgroup)
# nova secgroup-add-rule default icmp -1 -1 0.0.0.0/0
SecGroup.add_rule_to_db(group=secgroup,
name="ssh",
from_port="22",
to_port="22",
protocol="tcp",
cidr="0.0.0.0/0")
SecGroup.add_rule_to_db(group=secgroup,
name="http",
from_port="80",
to_port="80",
protocol="tcp",
cidr="0.0.0.0/0")
SecGroup.add_rule_to_db(group=secgroup,
name="https",
from_port="443",
to_port="443",
protocol="tcp",
cidr="0.0.0.0/0")
SecGroup.add_rule_to_db(group=secgroup,
name="icmp",
from_port="-1",
to_port="-1",
protocol="icmp",
cidr="0.0.0.0/0")
@classmethod
def delete(cls,
category='general',
group=None,
name=None):
# name is anme of the rule
if category=='general':
if name is None and group is not None:
# delete the entire group
cls.cm.delete(kind="secgrouprule", group=group)
elif name is not None and group is not None:
# delete specific rule
cls.cm.delete(name=name, kind="secgrouprule", group=group)
elif name is None and group is None:
# delete all groups
cls.cm.delete(kind="secgrouprule")
if group == Default.secgroup or Default.secgroup is None:
cls.reset_defaults()
else:
provider = CloudProvider(category).provider
# delete on cloud
if group is not None:
provider.delete_secgroup(name)
# delete the entire group
elif group is None:
# delete all groups
pass
@classmethod
def delete_secgroup(cls, name=None, cloud=None):
try:
# Find the secgroup from the cloud
cloud_provider = CloudProvider(cloud).provider
result = cloud_provider.delete_secgroup(name)
return result
except Exception as ex:
Console.error("delete group")
@classmethod
def delete_rule(cls, cloud, secgroup, from_port, to_port, protocol, cidr):
try:
args = {
"group": secgroup["uuid"],
"fromPort": from_port,
"toPort": to_port,
"protocol": protocol,
"cidr": cidr
}
rule = cls.cm.find(kind="secgrouprule",
output="object",
scope="first",
**args)
if rule is not None:
# get the nova client for cloud
cloud_provider = CloudProvider(cloud).provider
# delete the rule from the cloud
cloud_provider.delete_secgroup_rule(rule.uuid)
# delete the local db record
cls.cm.delete(rule)
return "Rule [{fromPort} | {toPort} | {protocol} | {cidr}] deleted" \
.format(**args)
else:
return None
except Exception as ex:
Console.error("delete rule")
return
@classmethod
def delete_all_rules(cls, secgroup):
try:
args = {
"group": secgroup["uuid"]
}
rules = cls.cm.find(kind="secgrouprule", output="object", **args)
if rules is not None:
for rule in rules:
cls.cm.delete(rule)
Console.ok("Rule [{fromPort} | {toPort} | {protocol} | {cidr}] deleted"
.format(**rule))
else:
pass
except Exception as ex:
Console.error("delete all rules")
return
# new methods moved from the test_secgroup:3
# the operations are from the perspective on the cloud
# and does not make any change on local db
#
@classmethod
def add_group_cloud(cls, cloud, groupname):
provider = CloudProvider(cloud).provider
return provider.create_secgroup(groupname)
@classmethod
def delete_group_cloud(cls, cloud, groupname):
provider = CloudProvider(cloud).provider
return provider.delete_secgroup(groupname)
@classmethod
def add_rule_cloud(cls, cloud, groupname, rulename):
ret = None
provider = CloudProvider(cloud).provider
# fetch rule from db
db_rule = cls.cm.find(kind="secgrouprule",
category="general",
group=groupname,
name=rulename,
scope='first',
output='dict')
kwargs = {}
kwargs["protocol"] = db_rule["protocol"]
kwargs["cidr"] = db_rule["cidr"]
kwargs["from_port"] = db_rule["fromPort"]
kwargs["to_port"] = db_rule["toPort"]
group = cls.get_group_cloud(cloud, groupname)
if group:
groupid = group["id"]
kwargs["uuid"] = groupid
ret = provider.add_secgroup_rule(**kwargs)
return ret
@classmethod
def delete_rule_cloud(cls, cloud, groupname, rulename):
ret = None
provider = CloudProvider(cloud).provider
ruleid = cls.get_rule_cloud(cloud, groupname, rulename)
if ruleid:
ret = provider.delete_secgroup_rule(ruleid)
#else:
# Console.error("Rule does not exist - Rule:{}, Group:{}"\
# .format(rulename, groupname), traceflag=False)
return ret
@classmethod
def delete_all_rules_cloud(cls, cloud, groupname):
rules = cls.list_rules_cloud(cloud, groupname)
provider = CloudProvider(cloud).provider
if rules:
for rule in rules:
ruleid = rule['id']
provider.delete_secgroup_rule(ruleid)
return
@classmethod
def list_groups_cloud(cls, cloud):
provider = CloudProvider(cloud).provider
groups = provider.list_secgroup(cloud)
return groups
@classmethod
def get_group_cloud(cls, cloud, groupname):
provider = CloudProvider(cloud).provider
groups = provider.list_secgroup(cloud)
ret = None
for groupkey in groups:
group = groups[groupkey]
if group["name"] == groupname:
ret = group
break
return ret
@classmethod
def list_rules_cloud(cls, cloud, groupname):
provider = CloudProvider(cloud).provider
groups = provider.list_secgroup(cloud)
for id in groups:
group = groups[id]
if groupname == group["name"]:
return group["rules"]
return None
@classmethod
def get_rule_cloud(cls, cloud, groupname, rulename):
rules = cls.list_rules_cloud(cloud, groupname)
# find properties for db rule
db_rule = cls.cm.find(kind="secgrouprule",
category="general",
group=groupname,
name=rulename,
scope='first',
output='dict')
ruleid = None
for rule in rules:
if 'cidr' in rule['ip_range']:
if (db_rule["fromPort"] == str(rule['from_port']) and
db_rule["toPort"] == str(rule['to_port']) and
db_rule["protocol"] == rule['ip_protocol'] and
db_rule["cidr"] == rule['ip_range']['cidr']
):
ruleid = rule['id'] #uuid for the rule
return ruleid
class Vm(ListResource):
cm = CloudmeshDatabase()
@classmethod
def generate_vm_name(cls, prefix=None, fill=3):
return Default.generate_name(Names.VM_COUNTER,
display_name=Default.user,
prefix=prefix,
fill=fill)
@classmethod
def uuid(cls, name, category=None):
vm = cls.get(name, category=category)
if vm is None:
return None
return vm.uuid
@classmethod
def get(cls, key, category=None):
"""
returns the value of the first objects matching the key
with the given category.
:param key: The dictionary key
:param category: The category
:return:
"""
if category is None:
o = cls.cm.find(kind='vm', output='dict', scope='first', name=key)
else:
o = cls.cm.find(category=category,
kind='vm',
output='dict',
scope='first',
name=key)
return o
@classmethod
def construct_ip_dict(cls, ip_addr, name=None):
# TODO kilo cloud as defualt should be avoided
if name is None:
Console.error("cloud name not set")
return None
try:
d = ConfigDict("cloudmesh.yaml")
cloud_details = d["cloudmesh"]["clouds"][name]
# Handle Openstack Specific Output
if cloud_details["cm_type"] == "openstack":
ipaddr = {}
for network in ip_addr:
index = 0
for ip in ip_addr[network]:
ipaddr[index] = {}
ipaddr[index]["network"] = network
ipaddr[index]["version"] = ip["version"]
ipaddr[index]["addr"] = ip["addr"]
index += 1
return ipaddr
# Handle EC2 Specific Output
if cloud_details["cm_type"] == "ec2":
# Console.TODO("ec2 ip dict yet to be implemented")
# TODO.implement()
# :type str: ip_addr
index = 0
ipaddr = {}
ipaddr[index] = {}
ipaddr[index]['addr'] = ip_addr
return ipaddr
# Handle Azure Specific Output
if cloud_details["cm_type"] == "azure":
index = 0
ipaddr = {}
for ip in ip_addr:
ipaddr[index] = {}
ipaddr[index]["network"] = ip
ipaddr[index]["version"] = 'ipv4'
ipaddr[index]["addr"] = ip
index += 1
return ipaddr
except Exception as e:
Console.error("error in vm construct dict %s" % e, traceflag=True)
@classmethod
def isUuid(cls, name):
try:
UUID(name, version=4)
return True
except ValueError:
return False
@classmethod
def boot(cls, **kwargs):
arg = dotdict(kwargs)
for a in ["key", "name", "image", "flavor"]:
if a not in kwargs:
raise ValueError(a + " not in arguments to vm boot")
conf = ConfigDict("cloudmesh.yaml")
arg.user = arg.user or conf["cloudmesh"]["profile"]["user"]
arg.group = arg.group or Default.group
cloud_provider = CloudProvider(arg.cloud).provider
if "nics" in arg:
nics = arg.nics
else:
nics = None
basic_dict = {
"cloud": arg.cloud,
"name": arg.name,
"image": arg.image,
"flavor": arg.flavor,
"key": arg.key,
"secgroup": [arg.secgroup],
"nics": nics,
"meta": {
'kind': 'cloudmesh',
'group': arg.group,
'cluster': arg.get('cluster', None),
'image': arg.image,
'flavor': arg.flavor,
'key': arg.key,
'category': arg.cloud
}
}
# Special case for Azure where certificate details needs to be added
if arg.cloud == "azure":
kwargs = dict()
kwargs['kind'] = "key_azure"
db_result = cls.cm.find(**kwargs)
# pprint("Key DB results")
key_result = None
try:
for key in db_result:
if key['name'] == arg.key:
pprint("Found the key")
key_result = key
break
if key_result is not None:
new_dict_items = dict()
new_dict_items['cert_thumbprint'] = key_result[
'fingerprint']
new_dict_items['pub_key_path'] = key_result['key_path']
new_dict_items['cert_path'] = key_result['certificate']
new_dict_items['pfx_path'] = key_result['pfx_path']
basic_dict.update(new_dict_items)
else:
pprint("None found in DB")
except:
traceback.print_exc()
pprint("Exception while processing azure boot arguments")
d = dotdict(basic_dict)
Console.ok(
"Machine {name} is being booted on cloud {cloud} ...".format(
**arg))
print(Printer.attribute(d))
vm = cloud_provider.boot_vm(**d)
if vm is not None:
cls.refresh(cloud=arg.cloud)
try:
# TODO: Repair db schema for vm_azure, vm_libcloud,
# vm_openstack. The following set only works with
# openstack, no libcloud, no azure
cls.cm.set(d.name, "key", d.key, scope="first", kind="vm")
cls.cm.set(d.name, "image", d.image, scope="first", kind="vm")
cls.cm.set(d.name,
"flavor",
d.flavor,
scope="first",
kind="vm")
cls.cm.set(d.name,
"group",
arg.group,
scope="first",
kind="vm")
cls.cm.set(d.name, "user", arg.user, scope="first", kind="vm")
cls.cm.set(d.name,
'username',
arg.username,
scope='first',
kind='vm')
cls.cm.set(d.name,
'cluster',
arg.cluster,
scope='first',
kind='vm')
except:
# cm.set error is identified as a warning, not an error
import sys
Console.warning("cls.cm.set error: %s" % (sys.exc_info()[0]))
# update group and key
#
# cls.cm.update("vm", name=data.name)
return vm
@classmethod
def start(cls, **kwargs):
arg = dotdict(kwargs)
cloud_provider = CloudProvider(arg.cloud).provider
for server in kwargs["servers"]:
cloud_provider.start_vm(server)
Console.ok("Machine {:} is being started on {:} Cloud...".format(
server, cloud_provider.cloud))
# Explicit refresh called after VM start, to update db.
# cls.refresh(cloud=kwargs["cloud"])
@classmethod
def stop(cls, **kwargs):
arg = dotdict(kwargs)
cloud_provider = CloudProvider(arg.cloud).provider
for server in kwargs["servers"]:
cloud_provider.stop_vm(server)
Console.ok("Machine {:} is being stopped on {:} Cloud...".format(
server, cloud_provider.cloud))
# Explicit refresh called after VM stop, to update db.
# cls.refresh(cloud=kwargs["cloud"])
@classmethod
def delete(cls, **kwargs):
arg = dotdict(kwargs)
force = kwargs.get("force", Default.purge)
if "cloud" in arg:
cloud_provider = CloudProvider(arg.cloud).provider
for server in kwargs["servers"]:
vm = cls.cm.find(name=server,
kind="vm",
cloud=arg.cloud,
scope="first")
#vm_by_id = cls.cm.find(cm_id=server, kind="vm", cloud=arg.cloud, scope="first")
#print (vm)
#print(vm_by_id)
#vm = vm or vm_by_id
if vm:
provider = vm["provider"]
cloud = vm["category"]
# If server has a floating ip associated, release it
server_dict = Network.get_instance_dict(
cloudname=arg.cloud, instance_id=server)
floating_ip = server_dict["floating_ip"]
if floating_ip is not None:
Network.disassociate_floating_ip(
cloudname=arg.cloud,
instance_name=server,
floating_ip=floating_ip)
cloud_provider.delete_vm(server)
if force:
cls.cm.delete(kind="vm",
provider=provider,
category=cloud,
name=server) # delete the record from db
Console.ok(
"VM record {:} is being deleted from the local database..."
.format(server))
else:
cls.cm.set(server,
"status",
"deleted",
kind="vm",
scope="first")
# Console.ok("VM {:} is being deleted on {:} cloud...".format(server, cloud_provider.cloud))
else:
Console.error("VM {:} can not be found.".format(server),
traceflag=False)
else:
clouds = set()
for server in arg.servers:
vm = cls.cm.find(kind="vm", name=server, scope="first")
if vm:
cloud = vm["category"]
provider = vm["provider"]
cloud_provider = CloudProvider(cloud).provider
clouds.add(cloud)
cloud_provider.delete_vm(server)
if force:
cls.cm.delete(kind="vm",
provider=provider,
category=cloud,
name=server)
Console.ok(
"VM record {:} is being deleted from the local database..."
.format(server))
else:
cls.cm.set(server,
"status",
"deleted",
kind="vm",
scope="first")
# Console.ok("VM {:} is being deleted on {:} cloud...".format(server, cloud))
else:
Console.error("VM {:} can not be found.".format(server),
traceflag=False)
@classmethod
def get_vms_by_name(cls, name, cloud):
vm_data = cls.cm.find(kind="vm", name=name, category=cloud)
if vm_data is None or len(vm_data) == 0:
raise RuntimeError("VM data not found in database.")
return vm_data
@classmethod
def get_vms_by_group(cls, name):
group = cls.cm.find(kind="group", name=name)
return group
@classmethod
def get_vm(cls, name):
vm = cls.cm.find(kind="vm", name=name)
return vm
@classmethod
def rename(cls, **kwargs):
arg = dotdict(kwargs)
cloud_provider = CloudProvider(kwargs["cloud"]).provider
# Check for vms with duplicate names in DB.
vms = cls.get_vms_by_name(name=arg.oldname, cloud=arg.cloud)
if len(vms) > 1:
users_choice = "y"
if not arg.force:
print("More than 1 vms found with the same name as {}.".format(
server))
users_choice = input(
"Would you like to auto-order the new names? (y/n): ")
if users_choice.strip() == "y":
count = 1
for index in vms:
count_new_name = "{0}{1}".format(arg.newname, count)
# print(vms[index])
cloud_provider.rename_vm(vms[index]["uuid"],
count_new_name)
print(
"Machine {0} with UUID {1} renamed to {2} on {3} cloud"
.format(vms[index]["name"], vms[index]["uuid"],
count_new_name, cloud_provider.cloud))
count += 1
elif users_choice.strip() == "n":
cloud_provider.rename_vm(arg.oldname, arg.newname)
print("Machine {0} renamed to {1} on {2} Cloud...".format(
arg.oldname, arg.newname, cloud_provider.cloud))
else:
Console.error("Invalid Choice.")
return
else:
cloud_provider.rename_vm(arg.oldname, arg.newname)
print("Machine {0} renamed to {1} on {2} Cloud...".format(
arg.oldname, arg.newname, cloud_provider.cloud))
# Explicit refresh called after VM rename, to update db.
cls.refresh(cloud=arg.cloud)
@classmethod
def info(cls, **kwargs):
raise NotImplementedError()
@classmethod
def list(cls, **kwargs):
"""
This method lists all VMs of the cloud
"""
arg = dotdict(kwargs)
if "name" in arg:
arg.name = arg.name
arg.output = arg.output or 'table'
# pprint (kwargs)
# prevent circular dependency
def vm_groups(vm):
"""
:param vm: name of the vm
:return: a list of groups the vm is in
"""
try:
query = {
'kind': "group",
'provider': 'general',
"species": "vm",
"member": vm,
"scope": 'all',
"output": 'dict'
}
d = cls.cm.find(**query)
groups_vm = set()
if d is not None and len(d) > 0:
for vm in d:
groups_vm.add(vm['name'])
return list(groups_vm)
except Exception as ex:
Console.error(ex.message)
return []
try:
if "name" in arg and arg.name is not None:
if cls.isUuid(arg.name):
elements = cls.cm.find(kind="vm",
category=arg.category,
uuid=arg.name)
else:
elements = cls.cm.find(kind="vm",
category=arg.category,
label=arg.name)
else:
elements = cls.cm.find(kind="vm", category=arg.category)
if elements is None or len(elements) == 0:
return None
for elem in elements:
element = elem
name = element["name"]
groups = vm_groups(name)
element["group"] = ','.join(groups)
# print(elements)
# order = ['id', 'uuid', 'name', 'cloud']
(order, header) = CloudProvider(arg.category).get_attributes("vm")
# order = None
if "name" in arg and arg.name is not None:
return Printer.attribute(elements[0], output=arg.output)
else:
return Printer.write(elements, order=order, output=arg.output)
except Exception as ex:
Console.error(ex.message)
@classmethod
def clear(cls, **kwargs):
raise NotImplementedError()
@classmethod
def refresh(cls, **kwargs):
# print("Inside refresh")
refreshed = cls.cm.refresh("vm", kwargs["cloud"])
# update counter
vms = cls.cm.find(kind='vm')
me = Default.user
for vm in vms:
name = vm['name']
if not name.startswith(me):
continue
number = name.split('-')[-1]
try:
# +1 as the stored counter is the next available counter
new_counter = int(number) + 1
except ValueError:
# name is not formatted correctly, possibly due to not
# being started using cloudmesh
continue
old_counter = Default.get_counter(Names.VM_COUNTER)
counter = max(new_counter, old_counter)
Default.set_counter(Names.VM_COUNTER, counter)
Console.debug_msg('Set counter ' + Names.VM_COUNTER + ' to ' +
str(Default.get_counter(Names.VM_COUNTER)))
return refreshed
@classmethod
def status_from_cloud(cls, **kwargs):
cloud_provider = CloudProvider(kwargs["cloud"]).provider
vm = cloud_provider.get_vm(name=kwargs["name"])
return vm["status"]
@classmethod
def set_login_user(cls, name=None, cloud=None, username=None):
# cls.cm.set(name, "username", username, kind="vm", scope="first")
vm = Vm.get(name, category=cloud)
if vm is None:
Console.error("VM could not be found", traceflag=False)
return
else:
cls.cm.update(kind="vm",
provider=vm["provider"],
filter={'name': name},
update={"username": username})
@classmethod
def get_login_user(cls, name, cloud):
print(name, cloud)
Console.error("this method is wrong implemented")
'''
if cls.isUuid(name):
uuid = name
else:
vm_data = cls.cm.find(kind="vm", category=cloud, label=name)
if vm_data is None or len(vm_data) == 0:
raise RuntimeError("VM with label {} not found in database.".format(name))
uuid = list(vm_data.values())[0]["uuid"]
# print(uuid)
user_map_entry = cls.cm.find("VMUSERMAP", vm_uuid=uuid)
# print(user_map_entry)
if user_map_entry is None or len(user_map_entry) == 0:
return None
else:
return list(user_map_entry.values())[0]["username"]
'''
@classmethod
def get_vm_public_ip(cls, vm_name, cloud):
"""
:param vm_name: Name of the VM instance whose Public IP has to be retrieved from the DB
:param cloud: Libcloud supported Cloud provider name
:return: Public IP as a list
"""
public_ip_list = []
vms = cls.get_vms_by_name(vm_name, cloud)
keys = vms.keys()
if keys is not None and len(keys) > 0:
public_ip = vms[keys[0]]["public_ips"]
if public_ip is not None and public_ip != "":
public_ip_list.append(public_ip)
return public_ip_list
def __init__(self, user=None):
self.db = CloudmeshDatabase.CloudmeshDatabase(user)