def config(self, config_path='~/.cloudmesh/cloudmesh4.yaml'):
"""
Use `cloudmesh4.yaml` file to configure.
"""
self._conf = Config(config_path).get("data")
# Set DB provider. There should only be one.
db_provider = self._conf.get('default.db')
if db_provider == 'local':
db_path = self._conf.get('db.local.CMDATA_DB_FOLDER')
self._db = LocalDBProvider(db_path)
# Check for local storage provider.
storage_path = self._conf.get('service.local.CMDATA_STORAGE_FOLDER')
if storage_path:
self._providers['local'] = LocalStorageProvider(storage_path)
# Check for Azure provider.
az_conf = self._conf.get('service.azure')
if az_conf:
az_act = az_conf.get('credentials.AZURE_STORAGE_ACCOUNT')
az_key = az_conf.get('credentials.AZURE_STORAGE_KEY')
az_container = az_conf.get('container')
if az_act and az_key:
self._providers['azure'] = AzureStorageProvider(az_act, az_key, az_container)
# Set a default storage provider.
default_storage_provider = self._conf.get('default.service')
self._providers['default'] = self._providers[default_storage_provider]
def __init__(self):
self.config = Config()
self.data = self.config.data["cloudmesh"]["data"]["mongo"]
self.expanduser()
pprint(self.config.dict())
def set_cloud(self, cloud):
"""
switch to another cloud provider
:param cloud: target provider
:return:
"""
self.cloud = cloud
self.os_config = Config().get('cloud.{}'.format(cloud))
def test_set(self):
before = self._conf.get("default.cloud")
self._conf.set("default.cloud", "testcloud")
new_config = Config()
after = new_config.get("default.cloud")
assert before != after
new_config.set("default.cloud", before)
def __init__(self, cloud=None):
config = Config().data
self.cloud = cloud
self.driver = None
self.key = None
if cloud:
self.os_config = config.get('cloudmesh').get('cloud').get(cloud)
self.driver = self.get_driver(cloud)
self.key = self.os_config.get('credentials').get(
'OS_KEY_PATH') # credentials.target return null string
# if we don't find OS_KEY_PATH in yaml, go to os.environ instead which can be set in .bashrc
if self.key is None:
self.key = os.environ['OS_KEY_PATH']
else:
self.os_config = config
def info(self, name=None):
"""
gets the information of a node with a given name
:param name:
:return: The dict representing the node including updated status
"""
arg = dotdict()
arg.name = name
config = Config()
cloud = "vagrant" # TODO: must come through parameter or set cloud
arg.path = config.data["cloudmesh"]["cloud"]["vagrant"]["default"][
"path"]
arg.directory = os.path.expanduser("{path}/{name}".format(**arg))
result = Shell.execute("vagrant", ["ssh-config"], cwd=arg.directory)
lines = result.split("\n")
data = {}
for line in lines:
attribute, value = line.strip().split(" ", 1)
if attribute == "IdentityFile":
value = value.replace('"', '')
data[attribute] = value
return data
def __init__(self):
config = Config().data['cloudmesh']
self.private_key_file = config['cloud']['aws']['credentials']['EC2_PRIVATE_KEY_FILE_PATH']
self.mongo = MongoDB(host=config['data']['mongo']['MONGO_HOST'],
username=config['data']['mongo']['MONGO_USERNAME'],
password=config['data']['mongo']['MONGO_PASSWORD'],
port=config['data']['mongo']['MONGO_PORT'])
def setup(self):
self.status_id = "test-id-1"
config = Config().data["cloudmesh"]["data"]["mongo"]
self.mongo = MongoDB(host=config["MONGO_HOST"],
username=config["MONGO_USERNAME"],
password=config["MONGO_PASSWORD"],
port=config["MONGO_PORT"])
class TestConfig:
def setup(self):
self.config = Config()
def test_00_config(self):
HEADING(myself())
pprint(self.config.dict())
print(self.config)
print(type(self.config.data))
#pprint(config.credentials('local'))
assert self.config is not None
#assert 'cloud' in config.cloud
def test_10_config_print(self):
HEADING(myself())
print(self.config)
assert True is True
def test_20_config_subscriptable(self):
HEADING(myself())
data = self.config["cloudmesh"]["data"]["mongo"]
assert data is not None
def __init__(self, debug):
"""
Initializes the SlurmCluster class
:param debug: switch the debug information on and off
"""
current_path = os.path.dirname(os.path.realpath(__file__))
self.workspace = os.path.join(current_path,
"batch_workspace/slurm_batch.yaml")
if not os.path.exists(os.path.dirname(self.workspace)):
os.makedirs(os.path.dirname(self.workspace))
self.cm_config = Config()
self.batch_config = GenericConfig(self.workspace)
self.debug = debug
self.all_jobIDs = []
self.slurm_cluster = {}
self.job_metadata = {}
def __init__(self, debug):
"""
Initializes the virtualcluster class
:param debug: switch the debug information on and off
"""
current_path = os.path.dirname(os.path.realpath(__file__))
self.workspace = os.path.join(current_path, "vcluster_workspace/vcluster.yaml")
if not os.path.exists(os.path.dirname(self.workspace)):
os.makedirs(os.path.dirname(self.workspace))
self.cm_config = Config()
self.vcluster_config = GenericConfig(self.workspace)
self.debug = debug
self.all_pids = []
self.virt_cluster = {}
self.runtime_config = {}
self.job_metadata = {}
def __init__(self):
"""
Initialization of the MOngo installer
"""
self.config = Config()
self.data = self.config.data["cloudmesh"]["data"]["mongo"]
self.expanduser()
def process_arguments(arguments):
"""
Process command line arguments to execute VM actions.
Called from cm4.command.command
:param arguments:
"""
config = Config()
default_cloud = config.data["cloudmesh"]["default"]["cloud"]
vm = Vm(default_cloud)
result = None
if arguments.get("--debug"):
pp = pprint.PrettyPrinter(indent=4)
print("vm processing arguments")
pp.pprint(arguments)
# pp.pprint(config.data)
if arguments.get("list"):
result = vm.nodes()
elif arguments.get("start"):
try:
result = vm.start(arguments.get("--vms"))
except ValueError:
vm_name = arguments.get("VMNAME")
vm.create(vm_name)
result = f"Created {vm_name}"
elif arguments.get("stop"):
result = vm.stop(arguments.get("--vms"))
elif arguments.get("destroy"):
result = vm.destroy(arguments.get("--vms"))
elif arguments.get("status"):
result = vm.status(arguments.get("--vms"))
elif arguments.get("publicip"):
result = vm.get_public_ips(arguments.get('--vms'))
elif arguments.get("ssh"):
# TODO
raise NotImplementedError(
"cm4 vm ssh command has not yet been implemented")
elif arguments.get("run"):
# TODO
raise NotImplementedError(
"cm4 vm run command has not yet been implemented")
elif arguments.get("script"):
# TODO
raise NotImplementedError(
"cm4 vm script command has not yet been implemented")
return result
def process_arguments(arguments):
provider = arguments['--provider']
config = Config()
if not provider:
provider = config.get("default.cloud")
if provider == "aws":
cloud_manager = AWSProvider(config)
else:
cloud_manager = AWSProvider(config)
if arguments['start']:
cloud_manager.start()
elif arguments['stop']:
cloud_manager.stop()
elif arguments['status']:
cloud_manager.status()
def __init__(self, host, username, password, port):
self.database = Config().data["cloudmesh"]["data"]["mongo"]["MONGO_DBNAME"]
self.host = host
self.password = urllib.parse.quote_plus(password)
self.username = urllib.parse.quote_plus(username)
self.port = port
self.client = None
self.db = None
self.connect_db()
def __init__(self, cloud):
self.config = Config().data["cloudmesh"]
self.provider = Vmprovider().get_provider(cloud)
self.mongo = MongoDB(
host=self.config["data"]["mongo"]["MONGO_HOST"],
username=self.config["data"]["mongo"]["MONGO_USERNAME"],
password=self.config["data"]["mongo"]["MONGO_PASSWORD"],
port=self.config["data"]["mongo"]["MONGO_PORT"])
def __init__(self, host=None, username=None, password=None, port=None):
config = Config().data["cloudmesh"]
self.database = config["data"]["mongo"]["MONGO_DBNAME"]
self.host = host or config["data"]["mongo"]["MONGO_HOST"]
self.password = urllib.parse.quote_plus(
password or config["data"]["mongo"]["MONGO_PASSWORD"])
self.username = urllib.parse.quote_plus(
username or config["data"]["mongo"]["MONGO_USERNAME"])
self.port = port or config["data"]["mongo"]["MONGO_PORT"]
self.client = None
self.db = None
self.connect_db()
def delete(self, name=None):
# TODO: check
arg = dotdict()
arg.name = name
config = Config()
cloud = "vagrant" # TODO: must come through parameter or set cloud
arg.path = config.data["cloudmesh"]["cloud"]["vagrant"]["default"][
"path"]
arg.directory = os.path.expanduser("{path}/{name}".format(**arg))
result = Shell.execute("vagrant", ["destroy", "-f", name],
cwd=arg.directory)
return result
class RestConfig(object):
config = Config().data["cloudmesh"]
MONGO_HOST = config["data"]["mongo"]["MONGO_HOST"]
MONGO_USERNAME = config["data"]["mongo"]["MONGO_USERNAME"]
MONGO_PASSWORD = config["data"]["mongo"]["MONGO_PASSWORD"]
MONGO_PORT = config["data"]["mongo"]["MONGO_PORT"]
MONGO_DBNAME = config["data"]["mongo"]["MONGO_DBNAME"]
# MONGO_DBNAME = config.get("data.mongo.MONGO_DBNAME")
# MONGO_HOST = config.get("data.mongo.MONGO_HOST")
# MONGO_PORT = config.get("data.mongo.MONGO_PORT")
# MONGO_USERNAME = config.get("data.mongo.MONGO_USERNAME")
# MONGO_PASSWORD = config.get("data.mongo.MONGO_PASSWORD")
MONGO_URI = "mongodb://" + MONGO_USERNAME + ":" + MONGO_PASSWORD + "@" + MONGO_HOST + ":" + MONGO_PORT + "/" + MONGO_DBNAME
class TestConfig:
"""
Functional tests for the configuration Config class
"""
def __init__(self):
self._conf = None
def setup(self):
self._conf = Config()
def test_get_notfound_defaults(self):
assert self._conf.get("nothere") is None
assert self._conf.get("nothere", {}) == {}
assert self._conf.get("default.nothere") is None
custom_default = {"foo": "bar"}
assert self._conf.get("default.nothere",
custom_default) == custom_default
def test_get_shorthand(self):
raw_result = self._conf._cloudmesh.get("default").get("cloud")
default_result = self._conf.get("default").get("cloud")
short_result = self._conf.get("default.cloud")
assert short_result == default_result == raw_result
az_conf = self._conf.get("cloud.azure")
az_id = az_conf.get('credentials.AZURE_SUBSCRIPTION_ID')
assert az_id is not None
def test_set(self):
before = self._conf.get("default.cloud")
self._conf.set("default.cloud", "testcloud")
new_config = Config()
after = new_config.get("default.cloud")
assert before != after
new_config.set("default.cloud", before)
def __init__(self, tenant_id, subscription_id, key, secret,
secure=True, host=None, port=None,
api_version=None, region=None, **kwargs):
self.config = Config().data["cloudmesh"]
self.defaults = self.config["cloud"]["azure"]["default"]
self.resource_group = self.defaults["resource_group"]
super().__init__(tenant_id=tenant_id,
subscription_id=subscription_id,
key=key, secret=secret,
secure=secure,
host=host, port=port,
api_version=api_version,
region=region, **kwargs)
def __init__(self, cloud):
self.mongo = MongoDB()
self.config = Config().data["cloudmesh"]
self.public_key_path = self.config["profile"]["key"]["public"]
self.kind = self.config["cloud"][cloud]["cm"]["kind"]
if self.kind == 'azure':
self.provider = AzureProvider(self.config)
elif self.kind == 'aws':
self.provider = AwsProvider(self.config)
elif self.kind == 'openstack':
self.provider = OpenstackCM("chameleon")
elif self.kind == 'vbox':
raise NotImplementedError
else:
raise NotImplementedError(f"Cloud `{self.kind}` not supported.")
def execute(self, name, command, cwd=None):
arg = dotdict()
arg.cwd = cwd
arg.command = command
arg.name = name
config = Config()
cloud = "vagrant" # TODO: must come through parameter or set cloud
arg.path = config.data["cloudmesh"]["cloud"]["vagrant"]["default"][
"path"]
arg.directory = os.path.expanduser("{path}/{name}".format(**arg))
vms = self.to_dict(self.nodes())
arg = "ssh {} -c {}".format(name, command)
result = Shell.execute("vagrant", ["ssh", name, "-c", command],
cwd=arg.directory)
return result
def _get_specification(self,
cloud=None,
name=None,
port=None,
image=None,
**kwargs):
arg = dotdict(kwargs)
arg.port = port
config = Config()
pprint(config.data)
if cloud is None:
#
# TOD read default cloud
#
cloud = "vagrant" # TODO must come through parameter or set cloud
print("CCC", cloud)
spec = config.data["cloudmesh"]["cloud"][cloud]
pprint(spec)
default = spec["default"]
pprint(default)
if name is not None:
arg.name = name
else:
# TODO get new name
pass
if image is not None:
arg.image = image
else:
arg.image = default["image"]
pass
arg.path = default["path"]
arg.directory = os.path.expanduser("{path}/{name}".format(**arg))
arg.vagrantfile = "{directory}/Vagrantfile".format(**arg)
return arg
def __init__(self, debug=False):
"""
TODO: doc
:param debug:
"""
# prepare path and directory
self.workspace = os.path.expanduser(
os.path.normpath(Config().data['cloudmesh']['cloud']['vbox']
['default']['vagrant_path']))
if not os.path.isdir(self.workspace):
self._nested_mkdir(self.workspace)
self.path = os.path.join(self.workspace, "Vagrantfile")
## if there is no Vagrantfile in the default Vagrantfile path, create one!
if not os.path.isfile(self.path):
self.create(count=2)
self.experiment_path = os.path.join(self.workspace, 'experiment')
if not os.path.isdir(self.experiment_path):
os.mkdir(self.experiment_path)
self.ssh_config = {}
self.debug = debug
from flask import request, jsonify
from cm4.mongo.mongoDB import MongoDB
from cm4.configuration.config import Config
from cm4.vm.Vm import Vm
config = Config()
db = MongoDB(config.get('data.mongo.MONGO_DBNAME'),
config.get('data.mongo.MONGO_USERNAME'),
config.get('data.mongo.MONGO_PASSWORD'),
config.get('data.mongo.MONGO_PORT'))
db.connect_db()
def vm_list():
cloud = request.args.get('cloud')
if cloud:
rep = Vm(cloud).list()
return 'No node is found on {}!\n'.format(cloud) if not rep else \
jsonify(**{'records': [db.var_to_json(x.__dict__) for x in rep]})
else:
return jsonify(**{'records': [db.var_to_json(x) for x in db.db['cloud'].find()]})
def setup(self):
self._conf = Config()
def setup(self):
self.config = Config()
self.azure = Vm('azure')
self.test_node_name = 'cm-test-vm-1'
self.test_node_id = ''
def __init__(self):
self.config = Config()
pprint(self.config.dict())
class OpenstackCM(CloudManagerABC):
# common
def __init__(self, cloud=None):
config = Config().data
self.cloud = cloud
self.driver = None
self.key = None
if cloud:
self.os_config = config.get('cloudmesh').get('cloud').get(cloud)
self.driver = self.get_driver(cloud)
self.key = self.os_config.get('credentials').get(
'OS_KEY_PATH') # credentials.target return null string
# if we don't find OS_KEY_PATH in yaml, go to os.environ instead which can be set in .bashrc
if self.key is None:
self.key = os.environ['OS_KEY_PATH']
else:
self.os_config = config
def _get_obj_list(self, obj_type):
obj_list = None
if obj_type == 'node':
obj_list = self.driver.list_nodes()
elif obj_type == 'image':
obj_list = self.driver.list_images()
elif obj_type == 'size':
obj_list = self.driver.list_sizes()
elif obj_type == 'ip':
obj_list = self.driver.ex_list_floating_ips()
return obj_list
def _get_obj_by_name(self, obj_type, obj_name):
obj_list = self._get_obj_list(obj_type)
for o in obj_list:
if o.name == obj_name:
return o
def _get_obj_by_id(self, obj_type, obj_id):
obj_list = self._get_obj_list(obj_type)
for o in obj_list:
if o.id == obj_id:
return o
def _get_node_by_id(self, node_id):
return self._get_obj_by_id('node', node_id)
def get_driver(self, cloud=None):
if not cloud:
raise ValueError('Cloud arguement is not properly configured')
if not self.driver:
self.driver = self.get_driver_helper(cloud)
return self.driver
def get_driver_helper(self, cloud):
credential = self.os_config.get("credentials")
openstack = get_driver(Provider.OPENSTACK)
driver = openstack(
credential.get('OS_USERNAME'),
credential.get('OS_PASSWORD') or os.environ['OS_PASSWORD'],
ex_force_auth_url=credential.get("OS_AUTH_URL"),
ex_force_auth_version='2.0_password',
ex_tenant_name=credential.get("OS_TENANT_NAME"),
ex_force_service_region=credential.get("OS_REGION_NAME"))
return driver
def set_cloud(self, cloud):
"""
switch to another cloud provider
:param cloud: target provider
:return:
"""
self.cloud = cloud
self.os_config = Config().get('cloud.{}'.format(cloud))
def _get_public_ip(self):
ips = [x for x in self._get_obj_list('ip') if not x.node_id]
# print(self._get_obj_list('ip'))
# print(ips[0].node_id)
return ips[0] if ips else None
# API hack for new VM class
def ex_start_node(self, node):
return self.driver.ex_start_node(node)
def ex_stop_node(self, info, deallocate):
return self.driver.ex_stop_node(info)
def destroy_node(self, node):
return self.driver.destroy_node(node)
def create_node(self, name):
return self.create(name)
def list_nodes(self):
return self.driver.list_nodes()
# APIs
def execute(self, name, command):
"""
execute arbitrary shell command on node through ssh
ssh funcionality must available on the local machine
:param name: name of the VM
:param command: shell command
"""
node = self._get_obj_by_name('node', name)
template = 'ssh -i {key} -o StrictHostKeyChecking=no {user}@{host} "{command}"'
kwargs = {
'key': os.path.splitext(self.key)[0],
'user': self.os_config.get('default.username'),
'host': node.public_ips[0],
'command': command
}
try:
res = subprocess.check_output(template.format(**kwargs),
shell=True,
input=b'\n',
stderr=subprocess.STDOUT)
return res.decode('utf8')
except Exception as e:
return e
def set_public_ip(self, name, ip_str):
"""
:param name: name of the VM
:param ip_str: ip string
"""
node = self._get_obj_by_name('node', name)
ip_obj = self.driver.ex_get_floating_ip(ip_str)
if ip_obj and not ip_obj.node_id:
self.driver.ex_attach_floating_ip_to_node(node, ip_obj)
elif ip_obj and ip_obj.node_id:
raise EnvironmentError(
'Public IP has been assigned to another machine. Pick another ip'
)
else:
raise ValueError('Public IP addresss does not exist!')
def remove_public_ip(self, name):
"""
:param name: name of the VM
"""
node = self._get_obj_by_name('node', name)
for ip in node.public_ips:
self.driver.ex_detach_floating_ip_from_node(node, ip)
# standard functions
def ls(self):
"""
list all nodes
:return: list of id, name, state
"""
nodes = self.driver.list_nodes()
return [dict(id=i.id, name=i.name, state=i.state) for i in nodes]
def list_available_ips(self):
index = 0
for x in self._get_obj_list('ip'):
if not x.node_id:
print("available ip_{}: {}".format(index, x))
index += 1
def nodes_info(self):
"""
get organized meta information about all node
:return: metadata of node
"""
nodes = self.driver.list_nodes()
res = {}
for i in nodes:
res[i.id] = dict(
id=i.id,
name=i.name,
state=i.state,
public_ips=i.public_ips,
private_ips=i.private_ips,
size=i.size,
image=i.image,
created_date=i.created_at.strftime("%Y-%m-%d %H:%M:%S"),
extra=i.extra)
return res
def info(self, node_id):
"""
get meta information about one node
:param node_id:
:return: metadata of node
"""
node = self._get_node_by_id(node_id)
return dict(id=node.id,
name=node.name,
state=node.state,
public_ips=node.public_ips,
private_ips=node.private_ips,
size=node.size,
image=node.image,
created_date=node.created_at.strftime("%Y-%m-%d %H:%M:%S"),
extra=node.extra)
def create(self, name, image=None, size=None, timeout=300, **kwargs):
# get defualt if needed
image_name = image if image else self.os_config.get('default').get(
'image')
size_name = size if size else self.os_config.get('default').get(
'flavor')
# add to kwargs
kwargs['name'] = name
kwargs['image'] = self._get_obj_by_name('image', image_name)
kwargs['size'] = self._get_obj_by_name('size', size_name)
if self.key:
try:
self.driver.import_key_pair_from_file(name, self.key)
except Exception as e:
print(e)
print(
"If exception code is 409 Conflict Key pair is already exists, we can still proceed without key importation"
)
kwargs['ex_keyname'] = name
# create node
node = self.driver.create_node(**kwargs)
# attach ip if available
# in case of error, need timeout to make sure we do attachment after the node has been spawned
ip = self._get_public_ip()
if ip:
timeout_counter = 0
while self.info(node.id)['state'] != 'running':
if timeout_counter > timeout:
print(
"Node is being spawned for too long, float ip association is failed"
)
return node
sleep(3)
timeout_counter += 3
self.driver.ex_attach_floating_ip_to_node(node, ip)
return node
def start(self, node_id):
"""
start the node
:param node_id:
:return: True/False
"""
node = self._get_node_by_id(node_id)
return self.driver.ex_start_node(node)
def stop(self, node_id):
"""
stop the node
:param node_id:
:return:
"""
node = self._get_node_by_id(node_id)
return self.driver.ex_stop_node(node)
def suspend(self, node_id):
"""
suspend the node
:param node_id:
:return: True/False
"""
node = self._get_node_by_id(node_id)
return self.driver.ex_suspend_node(node)
def resume(self, node_id):
"""
resume the node
:param node_id:
:return: True/False
"""
node = self._get_node_by_id(node_id)
return self.driver.ex_resume_node(node)
def reboot(self, node_id):
"""
resume the node
:param node_id:
:return: True/False
"""
node = self._get_node_by_id(node_id)
return self.driver.reboot_node(node)
def destroy(self, node_id):
"""
delete the node
:param node_id:
:return: True/False
"""
node = self._get_node_by_id(node_id)
return self.driver.destroy_node(node, )