class TestMongo:
def setup(self):
self.database = CmDatabase()
self.name = Name(experiment="exp",
group="grp",
user="******",
kind="vm",
counter=1)
def test_10_find_in_collection(self):
HEADING()
r = self.database.find_name("CC-CentOS7")
pprint(r)
def test_11_find_in_collections(self):
HEADING()
r = self.database.find_names("CC-CentOS7,CC-CentOS7-1811")
pprint(r)
def test_12_find_in_collection(self):
HEADING()
r = self.database.name_count("CC-CentOS7")
pprint(r)
class Provider(VolumeABC):
kind = "openstack"
sample = """
cloudmesh:
volume:
openstack:
cm:
active: true
heading: Chameleon
host: chameleoncloud.org
label: chameleon
kind: openstack
version: train
service: volume
credentials:
auth:
username: TBD
password: TBD
auth_url: https://kvm.tacc.chameleoncloud.org:5000/v3
project_id: TBD
project_name: cloudmesh
user_domain_name: Default
region_name: KVM@TACC
interface: public
identity_api_version: '3'
key_path: TBD/id_rsa.pub
default:
size: 1
volume_type: __DEFAULT__
"""
vm_state = [
'ACTIVE',
'BUILDING',
'DELETED',
'ERROR',
'HARD_REBOOT',
'PASSWORD',
'PAUSED',
'REBOOT',
'REBUILD',
'RESCUED',
'RESIZED',
'REVERT_RESIZE',
'SHUTOFF',
'SOFT_DELETED',
'STOPPED',
'SUSPENDED',
'UNKNOWN',
'VERIFY_RESIZE'
]
output = {
"volume": {
"sort_keys": ["cm.name"],
"order": ["cm.name",
"cm.cloud",
"cm.kind",
"availability_zone",
"created_at",
"size",
"status",
"id",
"volume_type"
],
"header": ["Name",
"Cloud",
"Kind",
"Availability Zone",
"Created At",
"Size",
"Status",
"Id",
"Volume Type"
],
}
}
def __init__(self, name):
"""
Initialize provider. The default parameters are read from the
configuration file that is defined in yaml format.
:param name: name of cloud
"""
self.cloud = name
self.config = Config()["cloudmesh.volume.openstack.credentials"]
self.defaults = Config()["cloudmesh.volume.openstack.default"]
self.cm = CmDatabase()
def update_dict(self, results):
"""
This function adds a cloudmesh cm dict to each dict in the list
elements. Typically this method is used
internally.
:param results: the original dicts.
:return: The list with the modified dicts
"""
if results is None:
return None
d = []
for entry in results:
volume_name = entry['name']
if "cm" not in entry:
entry['cm'] = {}
entry["cm"].update({
"cloud": self.cloud,
"kind": "volume",
"name": volume_name,
})
d.append(entry)
return d
def status(self, volume_name):
"""
This function get volume status, such as "in-use", "available"
:param volume_name: Volume name
:return: Volume_status
"""
con = openstack.connect(**self.config)
result = con.get_volume(name_or_id=volume_name)
result = [result]
result = self.update_dict(result)
return result
def list(self, **kwargs):
"""
This function list all volumes as following:
If NAME (volume_name) is specified, it will print out info of NAME
If NAME (volume_name) is not specified, it will print out info of all
volumes
:param kwargs: contains name of volume, vm name (optional)
:return: Dictionary of volumes
"""
try:
if kwargs and kwargs['refresh'] is False:
result = self.cm.find(cloud=self.cloud, kind='volume')
for key in kwargs:
if key == 'NAME' and kwargs['NAME']:
result = self.cm.find_name(name=kwargs['NAME'])
elif key == 'NAMES' and kwargs['NAMES']:
result = self.cm.find_names(names=kwargs['NAMES'])
else:
con = openstack.connect(**self.config)
results = con.list_volumes()
if kwargs and kwargs['NAME']:
result = con.get_volume(name_or_id=kwargs["NAME"])
result = [result]
result = self.update_dict(result)
if kwargs and kwargs['vm']:
server_id = con.get_server_id(name_or_id=kwargs['vm'])
vol_list = []
for entry in results:
attach_list = entry['attachments']
if len(attach_list) != 0:
if attach_list[0]['server_id'] == server_id:
vol_list.append(entry)
result = self.update_dict(vol_list)
else:
result = self.update_dict(results)
except Exception as e:
Console.error("Problem listing volumes", traceflag=True)
print(e)
raise RuntimeError
return result
def create(self, **kwargs):
"""
This function creates a new volume with default volume type __DEFAULT__.
Default parameters are read from self.config.
:param kwargs: Contains Volume name,size
:return: Volume dictionary
"""
try:
con = openstack.connect(**self.config)
arguments = dotdict(kwargs)
if arguments.volume_type is None:
arguments.volume_type = self.defaults["volume_type"]
if arguments.size is None:
arguments.size = self.defaults["size"]
r = con.create_volume(name=arguments.NAME,
size=arguments.size,
volume_type=arguments.volume_type
)
r = [r]
result = self.update_dict(r)
except Exception as e:
Console.error("Problem creating volume", traceflag=True)
print(e)
raise RuntimeError
return result
def attach(self, names=None, vm=None):
"""
This function attaches a given volume to a given instance
:param names: Names of Volumes
:param vm: Instance name
:return: Dictionary of volumes
"""
try:
con = openstack.connect(**self.config)
server = con.get_server(vm)
volume = con.get_volume(name_or_id=names[0])
con.attach_volume(server, volume, device=None, wait=True,
timeout=None)
except Exception as e:
Console.error("Problem attaching volume", traceflag=True)
print(e)
raise RuntimeError
return self.list(NAME=names[0], refresh=True)
def detach(self, name=None):
"""
This function detaches a given volume from an instance
:param name: Volume name
:return: Dictionary of volumes
"""
try:
con = openstack.connect(**self.config)
volume = con.get_volume(name_or_id=name)
attachments = volume['attachments']
server = con.get_server(attachments[0]['server_id'])
con.detach_volume(server, volume, wait=True, timeout=None)
except Exception as e:
Console.error("Problem detaching volume", traceflag=True)
print(e)
raise RuntimeError
# return of self.list(NAME=NAME)[0] throwing error:cm attribute
# not found inside CmDatabase.py. So manipulating result as below
t = self.list(NAME=name, refresh=True)[0]
result = {}
result.update(
{"cm": t["cm"],
"availability_zone": t["availability_zone"],
"created_at": t["created_at"],
"size": t["size"], "id": t["id"],
"status": t["status"],
"volume_type": t["volume_type"]
}
)
return result
def delete(self, name=None):
"""
This function delete one volume.
:param name: Volume name
:return: Dictionary of volumes
"""
try:
con = openstack.connect(**self.config)
con.delete_volume(name_or_id=name)
results = con.list_volumes()
result = self.update_dict(results)
except Exception as e:
Console.error("Problem deleting volume", traceflag=True)
print(e)
raise RuntimeError
return result
def add_tag(self, **kwargs):
"""
This function add tag to a volume.
:param kwargs:
NAME: name of volume
key: name of tag
value: value of tag
:return: Dictionary of volume
"""
try:
con = openstack.connect(**self.config)
name = kwargs['NAME']
key = kwargs['key']
value = kwargs['value']
metadata = {key: value}
con.update_volume(name_or_id=name, metadata=metadata)
except Exception as e:
Console.error("Problem in tagging volume", traceflag=True)
print(e)
raise RuntimeError
# return of self.list(NAME=NAME)[0] throwing error:cm attribute
# not found inside CmDatabase.py. So manipulating result as below
t = self.list(NAME=name, refresh=True)[0]
result = {}
result.update(
{"cm": t["cm"],
"availability_zone": t["availability_zone"],
"created_at": t["created_at"],
"size": t["size"], "id": t["id"],
"status": t["status"],
"volume_type": t["volume_type"]
}
)
return result
def migrate(self,
name=None,
fvm=None,
tvm=None,
fregion=None,
tregion=None,
fservice=None,
tservice=None,
fcloud=None,
tcloud=None,
cloud=None,
region=None,
service=None):
"""
Migrate volume from one vm to another vm.
:param name: name of volume
:param fvm: name of vm where volume will be moved from
:param tvm: name of vm where volume will be moved to
:param fregion: the region where the volume will be moved from
:param tregion: region where the volume will be moved to
:param fservice: the service where the volume will be moved from
:param tservice: the service where the volume will be moved to
:param fcloud: the provider where the volume will be moved from
:param tcloud: the provider where the volume will be moved to
:param cloud: the provider where the volume will be moved within
:param region: the region where the volume will be moved within
:param service: the service where the volume will be moved within
:return: dict
"""
raise NotImplementedError
def sync(self,
volume_id=None,
zone=None,
cloud=None):
"""
sync contents of one volume to another volume
:param volume_id: id of volume A
:param zone: zone where new volume will be created
:param cloud: the provider where volumes will be hosted
:return: str
"""
raise NotImplementedError
class Provider(VolumeABC):
kind = "volume"
sample = """
cloudmesh:
volume:
{name}:
cm:
active: '1'
heading: multipass
host: TBD
kind: multipass
version: TBD
service: volume
default:
path: /Volumes/multipass
"""
output = {
"volume": {
"sort_keys": ["cm.name"],
"order": [
"cm.name", "cm.cloud", "cm.kind", "State", "path",
'machine_path', "AttachedToVm", "tags", "time"
],
"header": [
"Name", "Cloud", "Kind", "State", "Path", 'Machine Path',
"AttachedToVm", "Tags", "Update Time"
]
}
}
def generate_volume_info(self, NAME, path):
"""
generate volume info dict.
info['AttachedToVm'] is a list of vm names where the volume is attached
to. (volume can attach to multiple vm and vm can have multiple
attachments) info['machine_path'] is the volume path in vm
info['time"] is the created time, will be updated as updated time
:param NAME: volume name
:param path: volume path
:return: dict
"""
info = {
'tags': [],
'name': NAME,
'path': path,
'AttachedToVm': [],
'State': 'available',
'machine_path': None,
'time': datetime.datetime.now()
}
return info
def update_volume_after_attached_to_vm(self, info, vms):
"""
Update volume info after attached to a vm.
info['AttachedToVm'] is a list of vm names where the volume is attached
to. info['machine_path'] is the volume path in vm
info['time"] is the updated as updated time
:param info: volume info got from MongoDB database
:param vms: attached to vms
:return: list of one dict
"""
path = info[0]['path']
path_list = path.split(sep='/')
machine_path_list = ["~", "Home"]
machine_path_list.extend(path_list[3:])
info[0]['machine_path'] = "/".join(machine_path_list)
info[0]['AttachedToVm'] = vms
info[0]['State'] = 'in-use'
info[0]['time'] = datetime.datetime.now()
return info
def update_volume_after_detach(self, info, vms):
"""
update volume info after detaching from a vm
info['AttachedToVm'] is a list of vm names where the volume is attached
to. info['time"] is the updated time
:param info: volume info
:param vms: attached to vms
:return: list of one dict
"""
info[0]['AttachedToVm'] = vms
if len(vms) == 0:
info[0]['machine_path'] = None
info[0]['State'] = 'available'
info[0]['time'] = datetime.datetime.now()
return info
def update_volume_tag(self, info, key, value):
"""
Update volume tag.
Tags is a key-value pair, with key as tag name and value as tag value,
tag = {key: value}. A volume can have multipale tags.
If given duplicated tag name, update the value to the current tag value.
:param value: value
:param key: key
:param info: volume info
:param vms: attached to vms
:return: list of one dict
"""
keys = []
for tag in info[0]['tags']:
if key == list(tag.keys())[0]:
if len(value) == 0:
info[0]['tags'].remove(tag)
keys.append(list(tag.keys())[0])
else:
tag.update({key: value})
keys.append(list(tag.keys())[0])
if key not in keys:
tag = {key: value}
info[0]['tags'].append(tag)
info[0]['time'] = datetime.datetime.now()
return info
def __init__(self, name):
"""
Initialize provider.
set cloudtype to "multipass", get the default dict, create a cloudmesh
database object.
:param name: name of cloud
"""
self.cloud = name
self.cloudtype = "multipass"
config = Config()
self.default = config[f"cloudmesh.volume.{self.cloud}.default"]
self.cm = CmDatabase()
def update_dict(self, elements, kind=None):
"""
converts the dict into a list.
:param elements: the list of original dicts. If elements is a single
dict a list with a single element is returned.
:param kind: "multipass"
:return: The list with the modified dicts
"""
if elements is None:
return None
d = []
for element in elements:
if "cm" not in element.keys():
element['cm'] = {}
element["cm"].update({
"kind": "volume",
"cloud": self.cloud,
"name": element['name'],
})
d.append(element)
return d
def create(self, **kwargs):
"""
This function create a new volume.
Default parameters from self.default, such as:
path="/Users/username/multipass".
Note: Windows users should also use "/" in file path.
:param NAME (string): the name of volume
:param path (string): path of volume
:return: dict
"""
for key in self.default.keys():
if key not in kwargs.keys():
kwargs[key] = self.default[key]
elif kwargs[key] is None:
kwargs[key] = self.default[key]
name = kwargs['NAME']
path = Path(kwargs['path'])
new_path = Path(f'{path}/{name}')
result = os.system(f"mkdir {new_path}")
if result == 0:
result = self.generate_volume_info(NAME=name, path=kwargs['path'])
result = self.update_dict([result])
return result
def delete(self, name):
"""
Delete volumes.
If name is not given, delete the most recent volume.
:param name: volume name
:return:
"""
result = self.cm.find_name(name)
path = result[0]['path']
delete_path = Path(f'{path}/{name}')
try:
os.system(f"rmdir {delete_path}")
result[0]['State'] = 'deleted'
result = self.update_dict(result)
except:
Console.error("volume is either not empty or not exist")
return result
def list(self, **kwargs):
"""
This function list all volumes as following:
If NAME (volume name) is specified, it will print out info of NAME.
If NAME (volume name) is not specified, it will print out info of all
volumes under current cloud.
If vm is specified, it will print out all the volumes attached to vm.
If region(path) is specified, it will print out
all the volumes in that region. i.e. /Users/username/multipass
:param NAME: name of volume
:param vm: name of vm
:param region: for multipass, it is the same with "path"
:return: dict
"""
if kwargs:
result = self.cm.find(cloud='multipass', kind='volume')
for key in kwargs:
if key == 'NAME' and kwargs['NAME']:
result = self.cm.find_name(name=kwargs['NAME'])
elif key == 'NAMES' and kwargs['NAMES']:
result = self.cm.find_names(names=kwargs['NAMES'])
elif key == 'vm' and kwargs['vm']:
result = self.cm.find(collection=f"{self.cloud}-volume",
query={'AttachedToVm': kwargs['vm']})
elif key == 'region' and kwargs['region']:
result = self.cm.find(collection=f"{self.cloud}-volume",
query={'path': kwargs['region']})
else:
result = self.cm.find(cloud='multipass', kind='volume')
return result
def _get_vm_status(self, name=None) -> dict:
"""
Get vm status.
:param name (string): vm name
:return: dict
"""
dict_result = {}
result = Shell.run(f"multipass info {name} --format=json")
if f'instance "{name}" does not exist' in result:
dict_result = {'name': name, 'status': "instance does not exist"}
else:
result = json.loads(result)
dict_result = {
'name': name,
'status': result["info"][name]['State']
}
return dict_result
def attach(self, names, vm):
"""
This function attach one or more volumes to vm. It returns info of
updated volume. The updated dict with "AttachedToVm" showing
the name of vm where the volume attached to.
:param names (string): names of volumes
:param vm (string): name of vm
:return: dict
"""
results = []
for name in names:
volume_info = self.cm.find_name(name)
if volume_info and volume_info[0]['State'] != "deleted":
vms = volume_info[0]['AttachedToVm']
path = volume_info[0]['path']
if vm in vms:
Console.error(f"{name} already attached to {vm}")
else:
result = self.mount(path=f"{path}/{name}", vm=vm)
mounts = result['mounts']
if f"{path}/{name}" in mounts.keys():
vms.append(vm)
result = self.update_volume_after_attached_to_vm(
info=volume_info, vms=vms)
results.append(result)
else:
Console.error(
"volume is not existed or volume had been deleted")
return results[0]
def mount(self, path=None, vm=None):
"""
mount volume to vm
:param path (string): path of volume
:param vm (string): name of vm
:return: dict
"""
os.system(f"multipass mount {path} {vm}")
dict_result = self._get_mount_status(vm=vm)
return dict_result
def _get_mount_status(self, vm=None):
"""
Get mount status of vm
:param vm (string): name of vm
:return:
"""
result = Shell.run(f"multipass info {vm} --format=json")
if f'instance "{vm}" does not exist' in result:
dict_result = {'name': vm, 'status': "instance does not exist"}
else:
result = json.loads(result)
dict_result = {
'name': vm,
'status': result["info"][vm]['state'],
'mounts': result["info"][vm]['mounts']
}
return dict_result
def unmount(self, path=None, vm=None):
"""
Unmount volume from vm
:param path (string): path of volume
:param vm (string): name of vm
:return:
"""
os.system(f"multipass unmount {vm}:{path}")
dict_result = self._get_mount_status(vm=vm)
return dict_result
def detach(self, name):
"""
This function detach a volume from vm. It returns the info of
the updated volume.
The vm under "AttachedToVm" will be removed if
volume is successfully detached.
Will detach volume from all vms.
:param name: name of volume to be detached
:return: dict
"""
volume_info = self.cm.find_name(name)
if volume_info and volume_info[0]['State'] != "deleted":
vms = volume_info[0]['AttachedToVm']
path = volume_info[0]['path']
if len(vms) == 0:
Console.error(f"{name} is not attached to any vm")
else:
removed = []
for vm in vms:
result = self.unmount(path=f"{path}/{name}", vm=vm)
mounts = result['mounts']
if f"{path}/{name}" not in mounts.keys():
removed.append(vm)
for vm in removed:
vms.remove(vm)
result = self.update_volume_after_detach(volume_info, vms)
return result[0]
else:
Console.error("volume does not exist or volume had been deleted")
def add_tag(self, **kwargs):
"""
This function add tag to a volume.
If volume name is not specified, then tag will be added to the last
volume.
:param NAME: name of volume
:param key: name of tag
:param value: value of tag
:return: dict
"""
key = kwargs['key']
value = kwargs['value']
volume_info = self.cm.find_name(name=kwargs['NAME'])
volume_info = self.update_volume_tag(info=volume_info,
key=key,
value=value)
return volume_info[0]
def status(self, name=None):
"""
This function get volume status, such as "in-use", "available",
"deleted"
:param name: volume name
:return: dict
"""
volume_info = self.cm.find_name(name)
if volume_info:
status = volume_info[0]['State']
else:
Console.error("volume is not existed")
return volume_info
def migrate(self, **kwargs):
"""
Migrate volume from one vm to another vm. "region" is volume path.
If vm and volume are in the same region (path), migrate within the same
region (path). If vm and volume are in different regions, migrate
between two regions (path)
:param NAME (string): the volume name
:param vm (string): the vm name
:return: dict
"""
volume_name = kwargs['NAME']
vm = kwargs['vm']
volume_info = self.cm.find_name(name=volume_name)
volume_attached_vm = volume_info[0]['AttachedToVm']
vm_info = Shell.run(f"multipass info {vm} --format=json")
vm_info = json.loads(vm_info)
vm_status = vm_info["info"][vm]['state']
if vm_status == 'running':
param = {'NAME': volume_name}
self.detach(**param)
self.attach(**param, vm=vm)
try:
for old_vm in volume_attached_vm:
volume_info[0]['AttachedToVm'].remove(old_vm)
except:
pass
volume_info[0]['AttachedToVm'].append(vm)
return volume_info
def sync(self, **kwargs):
"""
sync contents of one volume to another volume
:param names (list): list of volume names
:return: list of dict
"""
volume_1 = kwargs['NAMES'][0]
volume_2 = kwargs['NAMES'][1]
path1 = f"{self.cm.find_name(name=volume_1)[0]['path']}/{volume_1}/"
path2 = f"{self.cm.find_name(name=volume_2)[0]['path']}/{volume_2}/"
os.system(f"rsync -avzh {path2} {path1}")
kwargs1 = {'NAME': volume_1, 'key': "sync_with", 'value': volume_2}
volume_info1 = self.add_tag(**kwargs1)
result = [volume_info1]
return result
class Provider(VolumeABC):
kind = "oracle"
sample = """
cloudmesh:
volume:
{name}:
cm:
active: true
heading: {name}
host: TBD
label: {name}
kind: oracle
version: TBD
service: volume
credentials:
version: TBD
user: TBD
fingerprint: TBD
key_file: oci_api_key.pem
pass_phrase: TBD
tenancy: TBD
compartment_id: TBD
region: TBD
availability_domain: TBD
default:
"""
output = {
"volume": {
"sort_keys": ["cm.name"],
"order": ["cm.name",
"cm.cloud",
"cm.kind",
"availability_domain",
"time_created",
"size_in_gbs",
"lifecycle_state",
"id"
],
"header": ["Name",
"Cloud",
"Kind",
"Availability Zone",
"Created At",
"Size(Gb)",
"Status",
"Id"
],
}
}
def update_dict(self, results):
"""
This function adds a cloudmesh cm dict to each dict in the list
elements.
Libcloud
returns an object or list of objects With the dict method
this object is converted to a dict. Typically this method is used
internally.
:param results: the original dicts.
:return: The list with the modified dicts
"""
if results is None:
return None
d = []
for entry in results:
display_name = entry.__getattribute__("display_name")
availability_domain = entry.__getattribute__("availability_domain")
time_created = entry.__getattribute__("time_created")
size_in_gbs = entry.__getattribute__("size_in_gbs")
lifecycle_state = entry.__getattribute__("lifecycle_state")
attribute_id = entry.__getattribute__("id")
entry = {
"availability_domain": availability_domain,
"time_created": time_created,
"size_in_gbs": size_in_gbs,
"id": attribute_id,
"lifecycle_state": lifecycle_state
}
if "cm" not in entry:
entry['cm'] = {}
entry["cm"].update({
"cloud": self.cloud,
"kind": "volume",
"name": display_name,
})
d.append(entry)
return d
def __init__(self, name):
"""
Initialize provider. The default parameters are read from the
configuration file that is defined in yaml format.
:param name: name of cloud
"""
self.cloud = name
self.config = Config()["cloudmesh.volume.oracle.credentials"]
self.defaults = Config()["cloudmesh.volume.oracle.default"]
self.cm = CmDatabase()
def get_volume_id_from_name(self, block_storage, name):
"""
This function get volume id from volume name
:param block_storage: Block storage client object
:param name: volume name
:return: volume id
"""
v = block_storage.list_volumes(self.config['compartment_id'])
results = v.data
volume_id = None
for entry in results:
display_name = entry.__getattribute__("display_name")
if name == display_name:
volume_id = entry.__getattribute__("id")
break
return volume_id
def get_attachment_id_from_name(self, block_storage, name):
"""
This function get attachment id from volume name
:param block_storage: Block storage client object
:param name: Name of the volume
:return: Volume attachment id
"""
v = block_storage.list_volumes(self.config['compartment_id'])
results = v.data
attachment_id = None
for entry in results:
display_name = entry.__getattribute__("display_name")
if name == display_name:
tags = entry.__getattribute__("freeform_tags")
attachment_id = tags['attachment_id']
break
return attachment_id
def status(self, name):
"""
This function get volume status, such as "in-use", "available"
:param name: Volume name
:return: Volume_status
"""
try:
block_storage = oci.core.BlockstorageClient(self.config)
v = block_storage.list_volumes(self.config['compartment_id'])
volumes = v.data
result = []
entry = None
for entry in volumes:
display_name = entry.__getattribute__("display_name")
if name == display_name:
break
result.append(entry)
result = self.update_dict(result)
except Exception as e:
Console.error("Problem finding status", traceflag=True)
print(e)
raise RuntimeError
return result
def list(self, **kwargs):
"""
This function list all volumes as following:
If NAME (volume_name) is specified, it will print out info of NAME
If NAME (volume_name) is not specified, it will print out info of all
volumes
:param kwargs: contains name of volume
:return: Dictionary of volumes
"""
try:
if kwargs and kwargs['refresh'] is False:
result = self.cm.find(cloud=self.cloud, kind='volume')
for key in kwargs:
if key == 'NAME' and kwargs['NAME']:
result = self.cm.find_name(name=kwargs['NAME'])
elif key == 'NAMES' and kwargs['NAMES']:
result = self.cm.find_names(names=kwargs['NAMES'])
else:
block_storage = oci.core.BlockstorageClient(self.config)
if kwargs and kwargs['NAME']:
v = block_storage.list_volumes(
self.config['compartment_id'])
results = v.data
entry = None
for entry in results:
display_name = entry.__getattribute__("display_name")
if kwargs["NAME"] == display_name:
break
result = [entry]
result = self.update_dict(result)
else:
v = block_storage.list_volumes(
self.config['compartment_id'])
results = v.data
result = self.update_dict(results)
except Exception as e:
Console.error("Problem listing volume", traceflag=True)
print(e)
raise RuntimeError
return result
def create(self, **kwargs):
"""
This function creates a new volume with default size of 50gb.
Default parameters are read from self.config.
:param kwargs: Contains Volume name
:return: Volume dictionary
"""
try:
arguments = dotdict(kwargs)
block_storage = oci.core.BlockstorageClient(self.config)
result = block_storage.create_volume(
oci.core.models.CreateVolumeDetails(
compartment_id=self.config['compartment_id'],
availability_domain=self.config['availability_domain'],
display_name=arguments.NAME
))
# wait for availability of volume
oci.wait_until(
block_storage,
block_storage.get_volume(result.data.id),
'lifecycle_state',
'AVAILABLE'
).data
v = block_storage.list_volumes(self.config['compartment_id'])
results = v.data
result = self.update_dict(results)
except Exception as e:
Console.error("Problem creating volume", traceflag=True)
print(e)
raise RuntimeError
return result
def attach(self, names=None, vm=None):
"""
This function attaches a given volume to a given instance
:param names: Names of Volumes
:param vm: Instance name
:return: Dictionary of volumes
"""
try:
compute_client = oci.core.ComputeClient(self.config)
# get instance id from VM name
i = compute_client.list_instances(self.config['compartment_id'])
instances = i.data
instance_id = None
for entry in instances:
display_name = entry.__getattribute__("display_name")
if vm == display_name:
instance_id = entry.__getattribute__("id")
break
# get volumeId from Volume name
block_storage = oci.core.BlockstorageClient(self.config)
volume_id = self.get_volume_id_from_name(block_storage, names[0])
# attach volume to vm
a = compute_client.attach_volume(
oci.core.models.AttachIScsiVolumeDetails(
display_name='IscsiVolAttachment',
instance_id=instance_id,
volume_id=volume_id
)
)
# tag volume with attachment id. This needed during detach.
block_storage.update_volume(
volume_id,
oci.core.models.UpdateVolumeDetails(
freeform_tags={'attachment_id': a.data.id},
))
# wait until attached
oci.wait_until(
compute_client,
compute_client.get_volume_attachment(
a.data.id),
'lifecycle_state',
'ATTACHED'
)
# return result after attach
v = block_storage.list_volumes(self.config['compartment_id'])
results = v.data
results = self.update_dict(results)
except Exception as e:
Console.error("Problem attaching volume", traceflag=True)
print(e)
raise RuntimeError
return results
def detach(self, name=None):
"""
This function detaches a given volume from an instance
:param name: Volume name
:return: Dictionary of volumes
"""
try:
compute_client = oci.core.ComputeClient(self.config)
block_storage = oci.core.BlockstorageClient(self.config)
attachment_id = self.get_attachment_id_from_name(block_storage,
name)
compute_client.detach_volume(attachment_id)
# wait for detachment
oci.wait_until(
compute_client,
compute_client.get_volume_attachment(attachment_id),
'lifecycle_state',
'DETACHED'
)
# return result after detach
v = block_storage.list_volumes(self.config['compartment_id'])
results = v.data
results = self.update_dict(results)
except Exception as e:
Console.error("Problem detaching volume", traceflag=True)
print(e)
raise RuntimeError
return results[0]
def delete(self, name=None):
"""
This function delete one volume.
:param name: Volume name
:return: Dictionary of volumes
"""
try:
block_storage = oci.core.BlockstorageClient(self.config)
volume_id = self.get_volume_id_from_name(block_storage, name)
if volume_id is not None:
block_storage.delete_volume(volume_id=volume_id)
# wait for termination
oci.wait_until(
block_storage,
block_storage.get_volume(volume_id),
'lifecycle_state',
'TERMINATED'
).data
v = block_storage.list_volumes(self.config['compartment_id'])
results = v.data
result = self.update_dict(results)
except Exception as e:
Console.error("Problem deleting volume", traceflag=True)
print(e)
raise RuntimeError
return result
def add_tag(self, **kwargs):
"""
This function add tag to a volume.
:param kwargs:
NAME: name of volume
key: name of tag
value: value of tag
:return: Dictionary of volume
"""
try:
name = kwargs['NAME']
key = kwargs['key']
value = kwargs['value']
block_storage = oci.core.BlockstorageClient(self.config)
volume_id = self.get_volume_id_from_name(block_storage, name)
block_storage.update_volume(
volume_id,
oci.core.models.UpdateVolumeDetails(
freeform_tags={key: value},
)
)
result = self.list(NAME=name, refresh=True)[0]
except Exception as e:
Console.error("Problem adding tag", traceflag=True)
print(e)
raise RuntimeError
return result
def migrate(self,
name=None,
fvm=None,
tvm=None,
fregion=None,
tregion=None,
fservice=None,
tservice=None,
fcloud=None,
tcloud=None,
cloud=None,
region=None,
service=None):
"""
Migrate volume from one vm to another vm.
:param name: name of volume
:param fvm: name of vm where volume will be moved from
:param tvm: name of vm where volume will be moved to
:param fregion: the region where the volume will be moved from
:param tregion: region where the volume will be moved to
:param fservice: the service where the volume will be moved from
:param tservice: the service where the volume will be moved to
:param fcloud: the provider where the volume will be moved from
:param tcloud: the provider where the volume will be moved to
:param cloud: the provider where the volume will be moved within
:param region: the region where the volume will be moved within
:param service: the service where the volume will be moved within
:return: dict
"""
raise NotImplementedError
def sync(self,
volume_id=None,
zone=None,
cloud=None):
"""
sync contents of one volume to another volume
:param volume_id: id of volume A
:param zone: zone where new volume will be created
:param cloud: the provider where volumes will be hosted
:return: str
"""
raise NotImplementedError
class Provider(VolumeABC):
kind = "volume"
sample = """
cloudmesh:
volume:
{name}:
cm:
active: true
heading: AWS
host: aws.com
label: VAWAS1
kind: aws
version: TBD
service: volume
default:
volume_type: gp2
size: TBD
encrypted: False
region_name: {region_name}
region: {availability_zone}
snapshot: "None"
credentials:
EC2_SECURITY_GROUP: default
EC2_ACCESS_ID: {aws_access_key_id}
EC2_SECRET_KEY: {aws_secret_access_key}
EC2_PRIVATE_KEY_FILE_PATH: {private_key_file_path}
EC2_PRIVATE_KEY_FILE_NAME: {private_key_file_name}
"""
volume_status = ['in-use', 'available', 'creating', 'deleting']
output = {
"volume": {
"sort_keys": ["cm.name"],
"order": [
"cm.name",
"cm.cloud",
"cm.kind",
"cm.region",
# "AvailabilityZone",
# "CreateTime",
# "Encrypted",
"Size",
# "SnapshotId",
"State",
# "VolumeId",
# "Iops",
"cm.tags",
"VolumeType",
# "created",
"AttachedToVm",
# "UpdateTime"
],
"header": [
"Name",
"Cloud",
"Kind",
"Region",
# "AvailabilityZone",
# "Create Time",
# "Encrypted",
"Size(GB)",
# "SnapshotId",
"Status",
# "VolumeId",
# "Iops",
"Tags",
"Volume Type",
# "Created",
"Attached To Vm",
# "Update Time"
],
}
}
def __init__(self, name=None):
"""
Initialize provider, create boto3 ec2 client, get the default dict.
:param name: name of cloud
"""
self.cloud = name
config = Config()
self.default = config[f"cloudmesh.volume.{self.cloud}.default"]
self.cred = config[f'cloudmesh.volume.{self.cloud}.credentials']
self.client = boto3.client(
'ec2',
region_name=self.default['region_name'],
aws_access_key_id=self.cred['EC2_ACCESS_ID'],
aws_secret_access_key=self.cred['EC2_SECRET_KEY'])
self.cm = CmDatabase()
def update_dict(self, results):
"""
This function adds a cloudmesh cm dict to each dict in the list
elements. For aws, we make region = AvailabilityZone.
:param results: the original dicts.
:param kind: for volume special attributes are added. This includes
cloud, kind, name, region.
:return: The list with the modified dicts
"""
# {'Volumes':
# [
# {
# 'Attachments':
# [
# {
# 'AttachTime': datetime.datetime(2020, 3, 16, 20, 0,
# 35, tzinfo=tzutc()),
# 'Device': '/dev/sda1',
# 'InstanceId': 'i-0765529fec90ba56b',
# 'State': 'attached',
# 'VolumeId': 'vol-09db404935694e941',
# 'DeleteOnTermination': True
# }
# ],
# 'AvailabilityZone': 'us-east-2c',
# 'CreateTime': datetime.datetime(2020, 3, 16, 20, 0, 35,
# 257000, tzinfo=tzutc()),
# 'Encrypted': False,
# 'Size': 8,
# 'SnapshotId': 'snap-085c8383cc8833286',
# 'State': 'in-use',
# 'VolumeId': 'vol-09db404935694e941',
# 'Iops': 100,
# 'Tags':
# [{'Key': 'Name',
# 'Value': 'xin-vol-3'}],
# 'VolumeType': 'gp2'
# },
# {...}
# ]
# }
if results is None:
return None
d = []
elements = results['Volumes']
for entry in elements:
tags = ""
try:
tags = entry['Tags'].copy()
for item in entry['Tags']:
if item['Key'] == 'Name':
if item['Value'] == "":
# Console.error(f"Please name volume
# {entry['VolumeId']}")
volume_name = " "
elif item['Value'] == " ":
# Console.error(f"Please name volume
# {entry['VolumeId']}")
volume_name = " "
else:
volume_name = item['Value']
tags.remove(item)
else:
# Console.error(f"Please name volume
# {entry['VolumeId']}")
volume_name = " "
except:
# Console.error(f"Please name volume {entry['VolumeId']}")
volume_name = " "
if "cm" not in entry:
entry['cm'] = {}
entry["cm"].update({
"cloud": self.cloud,
"kind": "volume",
"name": volume_name,
"region": entry["AvailabilityZone"],
"tags": tags
})
d.append(entry)
return d
def vm_info(self, vm):
"""
This function find vm info through given vm name.
:param vm: the name of vm.
:return: dict
"""
vm_info = self.client.describe_instances(Filters=[
{
'Name': 'tag:Name',
'Values': [
vm,
]
},
])
return vm_info
def find_vm_info_from_volume_name(self, volume_name=None):
"""
This function find vm info which the volume attached to through given
volume name. Only implemented circumstance when a volume can only
attach to one vm. (type iol volume could attach to multiple vms,
not implemented)
:param volume_name: the name of volume.
:return: string
"""
volume = self.client.describe_volumes(Filters=[
{
'Name': 'tag:Name',
'Values': [
volume_name,
]
},
], )
elements = volume['Volumes']
for i in range(len(elements)):
try:
for item in elements[i]['Attachments']:
vm_id = item['InstanceId']
instance = client.describe_instances(InstanceIds=[vm_id])
for tag in instance['Reservations'][0]['Instances'][0][
'Tags']:
if tag['Key'] == 'Name':
vm_name = tag['Value']
return vm_name
except:
Console.error(f"{volume_name} does not attach to any vm")
def update_AttachedToVm(self, data):
"""
This function update returned volume dict with
result['Volumes'][i]['AttachedToVm'] = vm_name. "i" chould be more than
0 if volume could attach to multiple vm, but for now, one volume only
attach to one vm.
Only IOPS io1 volumes can attach to multiple vms (creating of io1 volume
is not implemented)
:param data: volume dict
:return: dict
"""
elements = data['Volumes']
for i in range(len(elements)):
elements[i]['AttachedToVm'] = []
try:
for item in elements[i]['Attachments']:
vm_id = item['InstanceId']
instance = self.client.describe_instances(
InstanceIds=[vm_id])
for tag in instance['Reservations'][0]['Instances'][0][
'Tags']:
if tag['Key'] == 'Name':
vm_name = tag['Value']
elements[i]['AttachedToVm'].append(vm_name)
except:
pass
return data
def find_volume_id(self, volume_name):
"""
This function find volume_id through volume_name
:param volume_name: the name of volume
:return: string
"""
volume = self.client.describe_volumes(Filters=[
{
'Name': 'tag:Name',
'Values': [
volume_name,
]
},
], )
volume_id = volume['Volumes'][0]['VolumeId']
return volume_id
def find_vm_id(self, vm_name):
"""
This function find vm_id through vm_name
:param vm_name: the name of vom
:return: string
"""
instance = self.client.describe_instances(Filters=[
{
'Name': 'tag:Name',
'Values': [
vm_name,
]
},
], )
vm_id = instance['Reservations'][0]['Instances'][0]['InstanceId']
return vm_id
def wait(self, time=None):
"""
This function waiting for volume to be updated
:param time: time to wait in seconds
:return: False
"""
Console.info("waiting for volume to be updated")
sleep(time)
return False
def status(self, name):
"""
This function get volume status, such as "in-use", "available",
"deleting"
:param name
:return: dict
"""
result = self.client.describe_volumes(Filters=[
{
'Name': 'tag:Name',
'Values': [
name,
]
},
], )
result = self.update_dict(result)
# volume_status = volume['Volumes'][0]['State']
return result
def create(self, **kwargs):
"""
This function create a new volume, with defalt parameters in
self.default.
default:
{volume_type: gp2
size: 2
encrypted: False
region: us-east-2a
snapshot: "None"}
:param NAME (string): the name of volume
:param size (int): the size of volume (GB)
:param volume_type: volume type
:param region (string): availability zone of volume
:return: dict
"""
for key in self.default.keys():
if key not in kwargs.keys():
kwargs[key] = self.default[key]
elif kwargs[key] is None:
kwargs[key] = self.default[key]
result = self._create(**kwargs)
result = self.update_dict(result)
return result
def _create(self, **kwargs):
"""
Create a volume.
:param name (string): name of volume
:param region (string): availability-zone
:param encrypted (boolean): True|False
:param size (integer): size of volume. Minimum size for st1 and sc1 is 500 GB.
:param volume_type (string): type of volume. This can be gp2 for General
Purpose SSD, io1 for Provisioned IOPS SSD
(not implemented), st1 for Throughput
Optimized HDD, sc1 for Cold HDD,
or standard for Magnetic volumes.
:param snapshot (string): snapshot id
:return: dict
"""
if kwargs['volume_type'] == 'io1':
raise NotImplementedError
if kwargs['volume_type'] in ['sc1', 'st1']:
if int(kwargs['size']) < 500:
Console.error("minimum volume size for sc1 is 500 GB")
if kwargs['snapshot'] != "None":
r = self.client.create_volume(
AvailabilityZone=kwargs['region'],
Encrypted=kwargs['encrypted'],
Size=int(kwargs['size']),
SnapshotId=kwargs['snapshot'],
VolumeType=kwargs['volume_type'],
TagSpecifications=[
{
'ResourceType': 'volume',
'Tags': [
{
'Key': "Name",
'Value': kwargs['NAME']
},
]
},
],
)
else:
r = self.client.create_volume(
AvailabilityZone=kwargs['region'],
Encrypted=kwargs['encrypted'],
Size=int(kwargs['size']),
VolumeType=kwargs['volume_type'],
TagSpecifications=[
{
'ResourceType': 'volume',
'Tags': [
{
'Key': "Name",
'Value': kwargs['NAME']
},
]
},
],
)
r = [r]
result = {'Volumes': r}
result['Volumes'][0]['AttachedToVm'] = []
return result
def list(self, **kwargs):
"""
This function list all volumes as following:
If NAME (volume name) is specified, it will print out info of NAME.
If NAME (volume name) is not specified, it will print out info of all
volumes under current cloud.
If vm is specified, it will print out all the volumes attached to vm.
If region(availability zone) is specified, it will print out
all the volumes in that region.
:param NAME: name of volume
:param vm: name of vm
:param region: name of availability zone
:return: dict of volume
"""
if kwargs and kwargs['refresh']:
result = self.client.describe_volumes()
for key in kwargs:
if key == 'NAME' and kwargs['NAME']:
result = self.client.describe_volumes(
# DryRun=dryrun,
Filters=[
{
'Name': 'tag:Name',
'Values': [
kwargs['NAME'],
]
},
], )
elif key == 'NAMES' and kwargs['NAMES']:
if type(kwargs['NAMES']) == str:
kwargs['NAMES'] = [kwargs['NAMES']]
result = self.client.describe_volumes(
# DryRun=dryrun,
Filters=[
{
'Name': 'tag:Name',
'Values': kwargs['NAMES'],
},
], )
elif key == 'vm' and kwargs['vm']:
vm_id = self.find_vm_id(kwargs['vm'])
result = self.client.describe_volumes(
# DryRun=dryrun,
Filters=[
{
'Name': 'attachment.instance-id',
'Values': [
vm_id,
]
},
], )
elif key == 'region' and kwargs['region']:
result = self.client.describe_volumes(
# DryRun=dryrun,
Filters=[
{
'Name': 'availability-zone',
'Values': [
kwargs['region'],
]
},
], )
result = self.update_AttachedToVm(result)
result = self.update_dict(result)
elif kwargs and not kwargs['refresh']:
result = self.cm.find(cloud=self.cloud, kind='volume')
for key in kwargs:
if key == 'NAME' and kwargs['NAME']:
result = self.cm.find_name(name=kwargs['NAME'])
elif key == 'NAMES' and kwargs['NAMES']:
result = self.cm.find_names(names=kwargs['NAMES'])
elif key == 'vm' and kwargs['vm']:
result = self.cm.find(collection=f"{self.cloud}-volume",
query={'AttachedToVm': kwargs['vm']})
elif key == 'region' and kwargs['region']:
result = self.cm.find(
collection=f"{self.cloud}-volume",
query={'AvailabilityZone': kwargs['region']})
else:
result = self.client.describe_volumes()
result = self.update_AttachedToVm(result)
result = self.update_dict(result)
return result
def delete(self, name):
"""
This function delete one volume.
It will return the info of volume with "state" updated as "deleted"
and will show in Database.
:param NAME (string): volume name
:return: dict
"""
result = self.client.describe_volumes(Filters=[
{
'Name': 'tag:Name',
'Values': [name]
},
], )
volume_id = self.find_volume_id(name)
if result['Volumes'][0]['State'] == 'available':
response = self.client.delete_volume(
VolumeId=volume_id) # noqa: F841
stop_timeout = 360
time = 0
while time <= stop_timeout:
sleep(5)
time += 5
try:
volume_status = self.status(
name=name)[0]['State'] # noqa: F841
except:
break
result['Volumes'][0]['State'] = 'deleted'
else:
Console.error("volume is not available")
result = self.update_dict(result)
return result
def attach(self, names, vm, dryrun=False):
"""
This function attach one or more volumes to vm. It returns self.list()
to list the updated volume. The updated dict with "AttachedToVm" showing
the name of vm where the volume attached to.
:param names (string): names of volumes
:param vm (string): name of vm
:param dryrun (boolean): True|False
:return: dict of volume
"""
devices = [
"/dev/sdb",
"/dev/sdd",
"/dev/sde",
"/dev/sdf",
"/dev/sdg",
"/dev/sdh",
]
vm_id = self.find_vm_id(vm)
for name in names:
volume_id = self.find_volume_id(name)
for device in devices:
try:
response = self.client.attach_volume(
Device=device,
InstanceId=vm_id,
VolumeId=volume_id,
DryRun=dryrun) # noqa: F841
except:
pass
return self.list(NAMES=names, refresh=True)
def detach(self, name):
"""
This function detach a volume from vm. It returns volume dict of
the updated volume. The vm under "AttachedToVm" will be removed if
volume is successfully detached.
:param name: name of volume to detach
:return: dict of volume
"""
volume_status = self.status(name=name)[0]['State']
if volume_status == 'in-use':
volume_id = self.find_volume_id(volume_name=name)
rresponse = self.client.detach_volume(
VolumeId=volume_id) # noqa: F841
stop_timeout = 360
time = 0
while time <= stop_timeout:
sleep(5)
time += 5
volume_status = self.status(name=name)[0]['State']
if volume_status == "available":
break
return self.list(NAME=name, refresh=True)[0]
def add_tag(self, **kwargs):
"""
This function add tag to a volume.
In aws Boto3, key for volume name is "Name". For example,
key="Name", value="user-volume-1".
It could also be used to rename or name a volume.
If NAME is not specified, then tag will be added to the last volume.
:param NAME: name of volume
:param key: name of tag
:param value: value of tag
:return: dict
"""
key = kwargs['key']
value = kwargs['value']
volume_id = self.find_volume_id(volume_name=kwargs['NAME'])
re = self.client.create_tags(
Resources=[
volume_id,
],
Tags=[
{
'Key': key,
'Value': value
},
],
)
if key == 'Name':
result = self.list(NAME=value, refresh=True)[0]
else:
result = self.list(NAME=kwargs['NAME'], refresh=True)[0]
return result
def migrate(self, **kwargs):
"""
Migrate volume from one vm to another vm.
:param NAME (string): the volume name
:param vm (string): the vm name
:param region (string): the availability zone
:return: dict of volume
"""
volume_name = kwargs['NAME']
vm = kwargs['vm']
volume_status = self.status(name=volume_name)[0]['State']
volume_region = self.list(NAME=volume_name,
refresh=True)[0]['cm']['region']
volume_id = self.find_volume_id(volume_name=volume_name)
vm_info = self.vm_info(vm=vm)
vm_status = vm_info['Reservations'][0]['Instances'][0]['State']['Name']
vm_region = vm_info['Reservations'][0]['Instances'][0]['Placement'][
'AvailabilityZone']
# vm_id = self.find_vm_id(vm_name=vm)
# migrate within same region:
if vm_status == 'running':
if volume_region == vm_region:
if volume_status == "in-use":
self.detach(name=volume_name)
self.attach(names=[
volume_name,
], vm=vm)
elif volume_status == "available":
self.attach(names=[
volume_name,
], vm=vm)
return self.list(NAME=volume_name, refresh=True)
else:
snapshot_id = self.client.create_snapshot(
VolumeId=volume_id, )['SnapshotId']
ec2 = boto3.resource('ec2')
snapshot = ec2.Snapshot(snapshot_id)
start_timeout = 360
time = 0
while time <= start_timeout:
sleep(5)
time += 5
if snapshot.state == "completed":
break
kwargs['snapshot'] = snapshot_id
kwargs['region'] = vm_region
new_volume = self.create(name=volume_name,
**kwargs) # noqa: F841
start_timeout = 360
time = 0
while time <= start_timeout:
sleep(5)
time += 5
status = self.status(name=volume_name)[0]['State']
if status == "available":
break
self.attach(names=[
volume_name,
], vm=vm)
response = self.client.delete_volume(
VolumeId=volume_id) # noqa: F841
else:
Console.error("vm is not available")
result = self.list(NAME=kwargs['NAME'], refresh=True)[0]
return result
def sync(self, **kwargs):
"""
sync contents of one volume to another volume
:param NAMES (list): list of volume names
:return: dict
"""
volume_1 = kwargs['NAMES'][0]
volume_1_region = self.list(NAME=volume_1,
refresh=True)[0]['cm']['region']
volume_2 = kwargs['NAMES'][0]
volume_2_id = self.find_volume_id(volume_name=volume_2)
snapshot_id = self.client.create_snapshot(
VolumeId=volume_2_id, )['SnapshotId']
ec2 = boto3.resource('ec2')
snapshot = ec2.Snapshot(snapshot_id)
start_timeout = 360
time = 0
while time <= start_timeout:
sleep(5)
time += 5
if snapshot.state == "completed":
break
self.delete(name=volume_1)
kwargs = {
'region': volume_1_region,
'snapshot': snapshot_id,
'NAME': volume_1
}
# new_volume = self.create(**kwargs)
start_timeout = 360
time = 0
while time <= start_timeout:
sleep(5)
time += 5
status = self.status(name=volume_1)[0]['State']
if status == "available":
break
return self.list(NAME=volume_1, refresh=True)[0]
class Provider(VolumeABC):
kind = "google"
sample = """
cloudmesh:
volume:
{name}:
cm:
active: true
heading: {name}
host: cloud.google.com
label: {name}
kind: google
version: v1
service: volume
default:
zone: us-central1-a
type: projects/{project_id}/zones/{zone}/diskTypes/pd-standard
sizeGb: 10
credentials:
project_id: {project_id}
path_to_service_account_json: ~/.cloudmesh/service_account.json
"""
output = {
"volume": {
"sort_keys": ["cm.name"],
"order": ["cm.name",
"cm.kind",
"cm.cloud",
"status",
"sizeGb",
"type",
"creationTimestamp",
"zone",
"users",
"description",
"labels"],
"header": ["Name",
"Kind",
"Cloud",
"Status",
"Size",
"Type",
"Created",
"Zone",
"Attached to VMs",
"Description",
"Tags"]
}
}
def __init__(self, name):
"""
Get Google Cloud credentials and defaults from cloudmesh.yaml and set
scopes for Google Compute Engine
:param name: name of cloud provider in cloudmesh.yaml file under
cloudmesh.volume
"""
self.cloud = name
config = Config()
self.cm = CmDatabase()
self.default = config[f"cloudmesh.volume.{name}.default"]
self.credentials = config[f"cloudmesh.volume.{name}.credentials"]
self.compute_scopes = [
'https://www.googleapis.com/auth/compute',
'https://www.googleapis.com/auth/cloud-platform',
'https://www.googleapis.com/auth/compute.readonly']
def _wait(self, time=None):
"""
This function waiting for volume to be updated
:param time: time to wait in seconds
"""
sleep(time)
def update_dict(self, elements):
"""
This function adds a cloudmesh cm dict to each dict in the list
elements. Typically this method is used internally.
:param elements: the list of original dicts. If elements is a single
dict a list with a single element is returned.
:return: The list with the modified dicts
"""
if elements is None:
return None
elif type(elements) == list:
_elements = elements
else:
_elements = [elements]
d = []
for entry in _elements:
if '/' in entry['type']:
entry['type'] = entry['type'].rsplit('/', 1)[1]
if '/' in entry['zone']:
entry['zone'] = entry['zone'].rsplit('/', 1)[1]
if 'targetLink' in entry:
name = entry['targetLink'].rsplit('/', 1)[1]
else:
name = entry['name']
if 'users' in entry:
_users = []
for user in entry['users']:
_users.append(user)
for item in _users:
if '/' in item:
_users = []
remove_user_url = user.rsplit('/', 1)[1]
_users.append(remove_user_url)
entry['users'] = _users
_labels = []
if 'labels' in entry:
for label in entry['labels']:
_labels.append(label)
entry['labels'] = _labels
if "cm" not in entry:
entry['cm'] = {}
entry["cm"].update({
"kind": 'volume',
"cloud": self.cloud,
"name": name,
"status": entry['status']
})
d.append(entry)
return d
def _get_credentials(self, path_to_service_account_file, scopes):
"""
Method to get the credentials using the Service Account JSON file.
:param path_to_service_account_file: Service Account JSON File path.
:param scopes: Scopes needed to provision.
:return: credentials used to get compute service
"""
_credentials = service_account.Credentials.from_service_account_file(
filename=path_to_service_account_file,
scopes=scopes)
return _credentials
def _get_compute_service(self):
"""
Method to get google compute service v1.
:return: Google Compute Engine API
"""
service_account_credentials = self._get_credentials(
self.credentials['path_to_service_account_json'],
self.compute_scopes)
# Authenticate using service account.
if service_account_credentials is None:
print('Credentials are required')
raise ValueError('Cannot Authenticate without Credentials')
else:
compute_service = build('compute', 'v1',
credentials=service_account_credentials)
return compute_service
def _get_disk(self, zone, disk):
"""
Get the specified persistent disk from the cloud
:param zone: name of the zone in which the disk is located
:param disk: name of the disk
:return: a dict representing the disk
"""
compute_service = self._get_compute_service()
disk = compute_service.disks().get(
project=self.credentials["project_id"],
zone=zone,
disk=disk).execute()
return disk
def _list_instances(self, instance=None):
"""
Gets a list of available VM instances
:return: list of dicts representing VM instances
"""
compute_service = self._get_compute_service()
instance_list = compute_service.instances().aggregatedList(
project=self.credentials["project_id"],
orderBy='creationTimestamp desc').execute()
found_instances = []
items = instance_list["items"]
for item in items:
if "instances" in items[item]:
instances = items[item]["instances"]
if instance is not None:
for vm in instances:
if vm == instance:
found_instances.append(vm)
continue
else:
for vm in instances:
found_instances.append(vm)
return found_instances
def list(self, **kwargs):
"""
Retrieves an aggregated list of persistent disks with most recently
created disks listed first.
:return: an array of dicts representing the disks
"""
compute_service = self._get_compute_service()
if kwargs and kwargs['refresh'] is False:
result = self.cm.find(cloud=self.cloud, kind='volume')
for key in kwargs:
if key == 'NAME' and kwargs['NAME']:
result = self.cm.find_name(name=kwargs['NAME'])
elif key == 'NAMES' and kwargs['NAMES']:
result = self.cm.find_names(names=kwargs['NAMES'])
found = []
if kwargs['region'] is not None:
disk_list = compute_service.disks().list(
project=self.credentials['project_id'],
zone=kwargs['region'],
orderBy='creationTimestamp desc').execute()
if 'items' in disk_list:
disks = disk_list['items']
for disk in disks:
found.append(disk)
result = self.update_dict(found)
if kwargs['NAMES'] is not None or kwargs['vm'] is not None:
disk_list = compute_service.disks().aggregatedList(
project=self.credentials["project_id"],
orderBy='creationTimestamp desc').execute()
if kwargs['NAMES'] is not None:
items = disk_list["items"]
for item in items:
if "disks" in items[item]:
disks = items[item]["disks"]
for disk in disks:
if disk in kwargs['NAMES']:
found.append(disk)
if kwargs['vm'] is not None:
items = disk_list["items"]
for item in items:
if "disks" in items[item]:
disks = items[item]["disks"]
for disk in disks:
if 'users' in disk:
users = disk['users']
for user in users:
remove_user_url = user.rsplit('/', 1)[1]
if remove_user_url == kwargs['vm']:
found.append(disk)
else:
items = disk_list["items"]
for item in items:
if "disks" in items[item]:
disks = items[item]["disks"]
for disk in disks:
found.append(disk)
result = self.update_dict(found)
return result
elif kwargs and kwargs['refresh'] is True:
found = []
if kwargs['region'] is not None:
disk_list = compute_service.disks().list(
project=self.credentials['project_id'],
zone=kwargs['region'],
orderBy='creationTimestamp desc').execute()
if 'items' in disk_list:
disks = disk_list['items']
for disk in disks:
found.append(disk)
elif kwargs['NAMES'] is not None or kwargs['vm'] is not None:
disk_list = compute_service.disks().aggregatedList(
project=self.credentials["project_id"],
orderBy='creationTimestamp desc').execute()
if kwargs['NAMES'] is not None:
items = disk_list["items"]
for item in items:
if "disks" in items[item]:
disks = items[item]["disks"]
for disk in disks:
if disk in kwargs['NAMES']:
found.append(disk)
elif kwargs['vm'] is not None:
items = disk_list["items"]
for item in items:
if "disks" in items[item]:
disks = items[item]["disks"]
for disk in disks:
if 'users' in disk:
users = disk['users']
for user in users:
remove_user_url = user.rsplit('/', 1)[1]
if remove_user_url == kwargs['vm']:
found.append(disk)
else:
disk_list = compute_service.disks().aggregatedList(
project=self.credentials["project_id"],
orderBy='creationTimestamp desc').execute()
items = disk_list["items"]
for item in items:
if "disks" in items[item]:
disks = items[item]["disks"]
for disk in disks:
found.append(disk)
result = self.update_dict(found)
return result
else:
disk_list = compute_service.disks().aggregatedList(
project=self.credentials["project_id"],
orderBy='creationTimestamp desc').execute()
found = []
items = disk_list["items"]
for item in items:
if "disks" in items[item]:
disks = items[item]["disks"]
for disk in disks:
found.append(disk)
result = self.update_dict(found)
return result
def create(self, **kwargs):
"""
Creates a persistent disk in the specified project using the data in
the request.
:return: a list containing the newly created disk
"""
compute_service = self._get_compute_service()
volume_type = kwargs['volume_type']
size = kwargs['size']
description = kwargs['description']
zone = kwargs['region']
if volume_type is None:
volume_type = self.default["type"]
if size is None:
size = self.default["sizeGb"]
if zone is None:
zone = self.default['zone']
compute_service.disks().insert(
project=self.credentials["project_id"],
zone=self.default['zone'],
body={'type': volume_type,
'name': kwargs['NAME'],
'sizeGb': str(size),
'description': description}).execute()
new_disk = self._get_disk(self.default['zone'], kwargs['NAME'])
# wait for disk to finish being created
while new_disk['status'] != 'READY':
self._wait(1)
new_disk = self._get_disk(zone, kwargs['NAME'])
update_new_disk = self.update_dict(new_disk)
return update_new_disk
def delete(self, name=None):
"""
Deletes the specified persistent disk.
Deleting a disk removes its data permanently and is irreversible.
:param name: Name of the disk to delete
"""
compute_service = self._get_compute_service()
disk_list = self.list()
zone = None
for disk in disk_list: # find disk in list and get zone
if disk['name'] == name:
zone = str(disk['zone'])
if zone is None:
banner(f'{name} was not found')
return
compute_service.disks().delete(
project=self.credentials["project_id"],
zone=zone,
disk=name).execute()
# attempt to call disk from cloud
try:
deleted_disk = self._get_disk(zone, name)
# wait for disk to be deleted if found in cloud
if deleted_disk['status'] == 'DELETING':
while deleted_disk['status'] == 'DELETING':
self._wait(1)
try:
deleted_disk = self._get_disk(zone, name)
except HttpError:
pass
except HttpError:
pass
def _get_instance(self, zone, instance):
"""
Get the specified instance from the cloud
:param zone: zone in which the instance is located
:param instance: name of the instance
:return: a dict representing the instance
"""
compute_service = self._get_compute_service()
vm = compute_service.instances().get(
project=self.credentials["project_id"],
zone=zone,
instance=instance).execute()
return vm
def _stop_instance(self, name=None, zone=None):
"""
stops the instance with the given name
:param name: name of the instance
:zone: zone in which the instance is located
"""
compute_service = self._get_compute_service()
compute_service.instances().stop(
project=self.credentials['project_id'],
zone=zone,
instance=name).execute()
vm = self._get_instance(zone, name)
# Wait for the instance to stop
while vm['status'] != 'TERMINATED':
self._wait(1)
vm = self._get_instance(zone, name)
def _start_instance(self, name=None, zone=None):
"""
starts the instance with the given name
:param name: name of the instance
:zone: zone in which the instance is located
"""
compute_service = self._get_compute_service()
compute_service.instances().start(
project=self.credentials['project_id'],
zone=zone,
instance=name).execute()
vm = self._get_instance(zone, name)
# Wait for the instance to start
while vm['status'] != 'RUNNING':
self._wait(1)
vm = self._get_instance(zone, name)
def attach(self, names, vm=None):
"""
Attach one or more disks to an instance. GCP requires that the
instance be stopped when attaching a disk. If the instance is running
when the attach function is called, the function will stop the instance
and then restart the instance after attaching the disk.
:param names: name(s) of disk(s) to attach
:param vm: instance name which the volume(s) will be attached to
:return: updated disks with current status
"""
compute_service = self._get_compute_service()
instance_list = self._list_instances()
zone_url = None
instance_status = None
for instance in instance_list:
if instance['name'] == vm:
zone_url = instance['zone']
instance_status = instance['status']
zone = zone_url.rsplit('/', 1)[1]
# Stop the instance if necessary
if instance_status == 'RUNNING':
banner(f"Stopping VM {vm}")
self._stop_instance(vm, zone)
# get URL source to disk(s) from list of disks
disk_list = self.list()
for name in names:
source = None
for disk in disk_list:
if disk['name'] == name:
source = disk['selfLink']
banner(f"Attaching {name}")
compute_service.instances().attachDisk(
project=self.credentials['project_id'],
zone=zone,
instance=vm,
body={'source': source,
'deviceName': name}).execute()
new_attached_disks = []
for name in names:
get_disk = self._get_disk(zone, name)
# wait for disk to finish attaching
while 'users' not in get_disk:
self._wait(1)
get_disk = self._get_disk(zone, name)
new_attached_disks.append(get_disk)
# update newly attached disks
result = self.update_dict(new_attached_disks)
# Restart the instance if previously running
if instance_status == 'RUNNING':
banner(f"Restarting VM {vm}")
self._start_instance(vm, zone)
return result
def detach(self, name=None):
"""
Detach a disk from all instances. GCP requires that the
instance be stopped when detaching a disk. If the instance is running
when the detach function is called, the function will stop the instance
and then restart the instance after detaching the disk.
:param name: name of disk to detach
:return: dict representing updated status of detached disk
"""
compute_service = self._get_compute_service()
instances = []
zone = None
disk_list = self.list()
for disk in disk_list:
if disk['name'] == name:
zone = disk['zone']
for user in disk['users']:
instances.append(user)
# detach disk from all instances
result = None
for instance in instances:
vm = self._get_instance(zone, instance)
instance_status = vm['status']
# Stop the instance if necessary
if instance_status == 'RUNNING':
banner(f"Stopping VM {instance}")
self._stop_instance(instance, zone)
banner(f"Detaching {name}")
compute_service.instances().detachDisk(
project=self.credentials['project_id'],
zone=zone,
instance=instance,
deviceName=name).execute()
# Wait for disk to detach
detached_disk = self._get_disk(zone, name)
if 'users' in detached_disk:
while instance in detached_disk['users']:
self._wait(1)
detached_disk = self._get_disk(zone, name)
# Restart the instance if necessary
if instance_status == 'RUNNING':
banner(f"Restarting VM {instance}")
self._start_instance(instance, zone)
# update newly detached disk
result = self.update_dict(detached_disk)
return result[0]
def add_tag(self, **kwargs):
"""
Add a key:value label to the disk
Unable to change the name of a disk in Google Cloud
:param kwargs: name of the disk with a key and a value for the label
:return: updated list of disks with new label
"""
compute_service = self._get_compute_service()
disk_list = self.list()
# find disk in list and get zone
zone = None
label_fingerprint = None
for disk in disk_list:
if disk['name'] == kwargs['NAME']:
zone = str(disk['zone'])
label_fingerprint = disk['labelFingerprint']
compute_service.disks().setLabels(
project=self.credentials['project_id'],
zone=zone,
resource=kwargs['NAME'],
body={'labelFingerprint': label_fingerprint,
'labels': {kwargs['key']: str(kwargs['value'])}}).execute()
tagged_disk = self._get_disk(self.default['zone'], kwargs['NAME'])
# wait for tag to be applied
while 'labels' not in tagged_disk:
self._wait(1)
tagged_disk = self._get_disk(self.default['zone'], kwargs['NAME'])
updated_disk = self.update_dict(tagged_disk)
return updated_disk[0]
def status(self, name=None):
"""
Get status of specified disk, such as 'READY'
:param name: name of disk
:return: list containing dict representing the disk
"""
disk_list = self.list()
vol = []
for disk in disk_list:
if disk['name'] == name:
vol.append(disk)
break
result = self.update_dict(vol)
return result
def migrate(self,
name=None,
from_vm=None,
to_vm=None):
"""
Migrate volume from one vm to another vm.
:param name: name of volume
:param from_vm: name of vm where volume will be moved from
:param to_vm: name of vm where volume will be moved to
:return: dict of disk with updated info
"""
self.detach(name)
self.attach(name, vm=to_vm)
result = self.status(name)
# this only would work when disk and instances are all in the same zone
# include how to migrate disks between zones and regions
raise NotImplementedError
def sync(self,
from_volume=None,
to_volume=None):
"""
Sync contents of one volume to another volume. It is a copy of all
changed content from one volume to the other.
:param from_volume: name of the from volume
:param to_volume: name of the to volume
:return: str
"""
# delete to_volume then recreate from source of from_volume?
raise NotImplementedError
