def info(self, name=None):
"""
gets the information of a node with a given name
:param name:
:return: The dict representing the node including updated status
"""
arg = dotdict()
arg.name = name
config = Config()
cloud = "vagrant" # TODO: must come through parameter or set cloud
arg.path = config.data["cloudmesh"]["cloud"]["vagrant"]["default"][
"path"]
arg.directory = os.path.expanduser("{path}/{name}".format(**arg))
result = Shell.execute("vagrant", ["ssh-config"], cwd=arg.directory)
lines = result.split("\n")
data = {}
for line in lines:
attribute, value = line.strip().split(" ", 1)
if attribute == "IdentityFile":
value = value.replace('"', '')
data[attribute] = value
return data
def __init__(self):
self.config = Config()
self.data = self.config.data["cloudmesh"]["data"]["mongo"]
self.expanduser()
pprint(self.config.dict())
def __init__(self):
config = Config().data['cloudmesh']
self.private_key_file = config['cloud']['aws']['credentials']['EC2_PRIVATE_KEY_FILE_PATH']
self.mongo = MongoDB(host=config['data']['mongo']['MONGO_HOST'],
username=config['data']['mongo']['MONGO_USERNAME'],
password=config['data']['mongo']['MONGO_PASSWORD'],
port=config['data']['mongo']['MONGO_PORT'])
def config(self, config_path='~/.cloudmesh/cloudmesh4.yaml'):
"""
Use `cloudmesh4.yaml` file to configure.
"""
self._conf = Config(config_path).get("data")
# Set DB provider. There should only be one.
db_provider = self._conf.get('default.db')
if db_provider == 'local':
db_path = self._conf.get('db.local.CMDATA_DB_FOLDER')
self._db = LocalDBProvider(db_path)
# Check for local storage provider.
storage_path = self._conf.get('service.local.CMDATA_STORAGE_FOLDER')
if storage_path:
self._providers['local'] = LocalStorageProvider(storage_path)
# Check for Azure provider.
az_conf = self._conf.get('service.azure')
if az_conf:
az_act = az_conf.get('credentials.AZURE_STORAGE_ACCOUNT')
az_key = az_conf.get('credentials.AZURE_STORAGE_KEY')
az_container = az_conf.get('container')
if az_act and az_key:
self._providers['azure'] = AzureStorageProvider(az_act, az_key, az_container)
# Set a default storage provider.
default_storage_provider = self._conf.get('default.service')
self._providers['default'] = self._providers[default_storage_provider]
def process_arguments(arguments):
version = cm4.__version__
arguments = dotdict(arguments)
if arguments.get("--version"):
print(version)
elif arguments.admin and arguments.mongo:
print("MONGO")
result = cm4.mongo.MongoDBController.process_arguments(arguments)
print(result)
elif arguments.get("vm"):
result = cm4.vm.Vm.process_arguments(arguments)
print(result)
elif arguments.get('aws'):
cm4.aws.CommandAWS.process_arguments(arguments)
elif arguments.get("vagrant"):
cm4.vagrant.vagrant_basic.process_arguments(arguments)
elif arguments.get("vcluster"):
cm4.vcluster.VirtualCluster.process_arguments(arguments)
elif arguments.get("batch"):
cm4.batch.Batch.process_arguments(arguments)
elif arguments.get("openstack"):
cm4.openstack.OpenstackCM.process_arguments(arguments)
elif arguments.get("data"):
cm4.data.data.process_arguments(arguments)
elif arguments.get("set"):
config = Config()
if arguments.get("cloud"):
config.set("default.cloud", arguments.get("CLOUD"))
elif arguments.get("group"):
config.set("default.group", arguments.get("GROUP"))
elif arguments.get("role"):
config.set("default.role", arguments.get("ROLE"))
elif arguments.get("cluster"):
config.set("default.cluster", arguments.get("CLUSTER"))
elif arguments.get("experiment"):
config.set("default.experiment", arguments.get("EXPERIMENT"))
elif arguments.get("host"):
config.set("default.host", arguments.get("HOST"))
elif arguments.get("--key") and arguments.get("--value"):
config.set(arguments.get("--key"), arguments.get("--value"))
print("Config has been updated.")
def setup(self):
self.status_id = "test-id-1"
config = Config().data["cloudmesh"]["data"]["mongo"]
self.mongo = MongoDB(host=config["MONGO_HOST"],
username=config["MONGO_USERNAME"],
password=config["MONGO_PASSWORD"],
port=config["MONGO_PORT"])
def __init__(self):
"""
Initialization of the MOngo installer
"""
self.config = Config()
self.data = self.config.data["cloudmesh"]["data"]["mongo"]
self.expanduser()
def set_cloud(self, cloud):
"""
switch to another cloud provider
:param cloud: target provider
:return:
"""
self.cloud = cloud
self.os_config = Config().get('cloud.{}'.format(cloud))
def process_arguments(arguments):
"""
Process command line arguments to execute VM actions.
Called from cm4.command.command
:param arguments:
"""
config = Config()
default_cloud = config.data["cloudmesh"]["default"]["cloud"]
vm = Vm(default_cloud)
result = None
if arguments.get("--debug"):
pp = pprint.PrettyPrinter(indent=4)
print("vm processing arguments")
pp.pprint(arguments)
# pp.pprint(config.data)
if arguments.get("list"):
result = vm.nodes()
elif arguments.get("start"):
try:
result = vm.start(arguments.get("--vms"))
except ValueError:
vm_name = arguments.get("VMNAME")
vm.create(vm_name)
result = f"Created {vm_name}"
elif arguments.get("stop"):
result = vm.stop(arguments.get("--vms"))
elif arguments.get("destroy"):
result = vm.destroy(arguments.get("--vms"))
elif arguments.get("status"):
result = vm.status(arguments.get("--vms"))
elif arguments.get("publicip"):
result = vm.get_public_ips(arguments.get('--vms'))
elif arguments.get("ssh"):
# TODO
raise NotImplementedError(
"cm4 vm ssh command has not yet been implemented")
elif arguments.get("run"):
# TODO
raise NotImplementedError(
"cm4 vm run command has not yet been implemented")
elif arguments.get("script"):
# TODO
raise NotImplementedError(
"cm4 vm script command has not yet been implemented")
return result
def __init__(self, host, username, password, port):
self.database = Config().data["cloudmesh"]["data"]["mongo"]["MONGO_DBNAME"]
self.host = host
self.password = urllib.parse.quote_plus(password)
self.username = urllib.parse.quote_plus(username)
self.port = port
self.client = None
self.db = None
self.connect_db()
def __init__(self, cloud):
self.config = Config().data["cloudmesh"]
self.provider = Vmprovider().get_provider(cloud)
self.mongo = MongoDB(
host=self.config["data"]["mongo"]["MONGO_HOST"],
username=self.config["data"]["mongo"]["MONGO_USERNAME"],
password=self.config["data"]["mongo"]["MONGO_PASSWORD"],
port=self.config["data"]["mongo"]["MONGO_PORT"])
def test_set(self):
before = self._conf.get("default.cloud")
self._conf.set("default.cloud", "testcloud")
new_config = Config()
after = new_config.get("default.cloud")
assert before != after
new_config.set("default.cloud", before)
def __init__(self, host=None, username=None, password=None, port=None):
config = Config().data["cloudmesh"]
self.database = config["data"]["mongo"]["MONGO_DBNAME"]
self.host = host or config["data"]["mongo"]["MONGO_HOST"]
self.password = urllib.parse.quote_plus(
password or config["data"]["mongo"]["MONGO_PASSWORD"])
self.username = urllib.parse.quote_plus(
username or config["data"]["mongo"]["MONGO_USERNAME"])
self.port = port or config["data"]["mongo"]["MONGO_PORT"]
self.client = None
self.db = None
self.connect_db()
class RestConfig(object):
config = Config().data["cloudmesh"]
MONGO_HOST = config["data"]["mongo"]["MONGO_HOST"]
MONGO_USERNAME = config["data"]["mongo"]["MONGO_USERNAME"]
MONGO_PASSWORD = config["data"]["mongo"]["MONGO_PASSWORD"]
MONGO_PORT = config["data"]["mongo"]["MONGO_PORT"]
MONGO_DBNAME = config["data"]["mongo"]["MONGO_DBNAME"]
# MONGO_DBNAME = config.get("data.mongo.MONGO_DBNAME")
# MONGO_HOST = config.get("data.mongo.MONGO_HOST")
# MONGO_PORT = config.get("data.mongo.MONGO_PORT")
# MONGO_USERNAME = config.get("data.mongo.MONGO_USERNAME")
# MONGO_PASSWORD = config.get("data.mongo.MONGO_PASSWORD")
MONGO_URI = "mongodb://" + MONGO_USERNAME + ":" + MONGO_PASSWORD + "@" + MONGO_HOST + ":" + MONGO_PORT + "/" + MONGO_DBNAME
def delete(self, name=None):
# TODO: check
arg = dotdict()
arg.name = name
config = Config()
cloud = "vagrant" # TODO: must come through parameter or set cloud
arg.path = config.data["cloudmesh"]["cloud"]["vagrant"]["default"][
"path"]
arg.directory = os.path.expanduser("{path}/{name}".format(**arg))
result = Shell.execute("vagrant", ["destroy", "-f", name],
cwd=arg.directory)
return result
def __init__(self, cloud=None):
config = Config().data
self.cloud = cloud
self.driver = None
self.key = None
if cloud:
self.os_config = config.get('cloudmesh').get('cloud').get(cloud)
self.driver = self.get_driver(cloud)
self.key = self.os_config.get('credentials').get(
'OS_KEY_PATH') # credentials.target return null string
# if we don't find OS_KEY_PATH in yaml, go to os.environ instead which can be set in .bashrc
if self.key is None:
self.key = os.environ['OS_KEY_PATH']
else:
self.os_config = config
def __init__(self, tenant_id, subscription_id, key, secret,
secure=True, host=None, port=None,
api_version=None, region=None, **kwargs):
self.config = Config().data["cloudmesh"]
self.defaults = self.config["cloud"]["azure"]["default"]
self.resource_group = self.defaults["resource_group"]
super().__init__(tenant_id=tenant_id,
subscription_id=subscription_id,
key=key, secret=secret,
secure=secure,
host=host, port=port,
api_version=api_version,
region=region, **kwargs)
def __init__(self, cloud):
self.mongo = MongoDB()
self.config = Config().data["cloudmesh"]
self.public_key_path = self.config["profile"]["key"]["public"]
self.kind = self.config["cloud"][cloud]["cm"]["kind"]
if self.kind == 'azure':
self.provider = AzureProvider(self.config)
elif self.kind == 'aws':
self.provider = AwsProvider(self.config)
elif self.kind == 'openstack':
self.provider = OpenstackCM("chameleon")
elif self.kind == 'vbox':
raise NotImplementedError
else:
raise NotImplementedError(f"Cloud `{self.kind}` not supported.")
def __init__(self, debug):
"""
Initializes the SlurmCluster class
:param debug: switch the debug information on and off
"""
current_path = os.path.dirname(os.path.realpath(__file__))
self.workspace = os.path.join(current_path,
"batch_workspace/slurm_batch.yaml")
if not os.path.exists(os.path.dirname(self.workspace)):
os.makedirs(os.path.dirname(self.workspace))
self.cm_config = Config()
self.batch_config = GenericConfig(self.workspace)
self.debug = debug
self.all_jobIDs = []
self.slurm_cluster = {}
self.job_metadata = {}
def __init__(self, debug):
"""
Initializes the virtualcluster class
:param debug: switch the debug information on and off
"""
current_path = os.path.dirname(os.path.realpath(__file__))
self.workspace = os.path.join(current_path, "vcluster_workspace/vcluster.yaml")
if not os.path.exists(os.path.dirname(self.workspace)):
os.makedirs(os.path.dirname(self.workspace))
self.cm_config = Config()
self.vcluster_config = GenericConfig(self.workspace)
self.debug = debug
self.all_pids = []
self.virt_cluster = {}
self.runtime_config = {}
self.job_metadata = {}
def execute(self, name, command, cwd=None):
arg = dotdict()
arg.cwd = cwd
arg.command = command
arg.name = name
config = Config()
cloud = "vagrant" # TODO: must come through parameter or set cloud
arg.path = config.data["cloudmesh"]["cloud"]["vagrant"]["default"][
"path"]
arg.directory = os.path.expanduser("{path}/{name}".format(**arg))
vms = self.to_dict(self.nodes())
arg = "ssh {} -c {}".format(name, command)
result = Shell.execute("vagrant", ["ssh", name, "-c", command],
cwd=arg.directory)
return result
def process_arguments(arguments):
provider = arguments['--provider']
config = Config()
if not provider:
provider = config.get("default.cloud")
if provider == "aws":
cloud_manager = AWSProvider(config)
else:
cloud_manager = AWSProvider(config)
if arguments['start']:
cloud_manager.start()
elif arguments['stop']:
cloud_manager.stop()
elif arguments['status']:
cloud_manager.status()
def _get_specification(self,
cloud=None,
name=None,
port=None,
image=None,
**kwargs):
arg = dotdict(kwargs)
arg.port = port
config = Config()
pprint(config.data)
if cloud is None:
#
# TOD read default cloud
#
cloud = "vagrant" # TODO must come through parameter or set cloud
print("CCC", cloud)
spec = config.data["cloudmesh"]["cloud"][cloud]
pprint(spec)
default = spec["default"]
pprint(default)
if name is not None:
arg.name = name
else:
# TODO get new name
pass
if image is not None:
arg.image = image
else:
arg.image = default["image"]
pass
arg.path = default["path"]
arg.directory = os.path.expanduser("{path}/{name}".format(**arg))
arg.vagrantfile = "{directory}/Vagrantfile".format(**arg)
return arg
def __init__(self, debug=False):
"""
TODO: doc
:param debug:
"""
# prepare path and directory
self.workspace = os.path.expanduser(
os.path.normpath(Config().data['cloudmesh']['cloud']['vbox']
['default']['vagrant_path']))
if not os.path.isdir(self.workspace):
self._nested_mkdir(self.workspace)
self.path = os.path.join(self.workspace, "Vagrantfile")
## if there is no Vagrantfile in the default Vagrantfile path, create one!
if not os.path.isfile(self.path):
self.create(count=2)
self.experiment_path = os.path.join(self.workspace, 'experiment')
if not os.path.isdir(self.experiment_path):
os.mkdir(self.experiment_path)
self.ssh_config = {}
self.debug = debug
from flask import request, jsonify
from cm4.mongo.mongoDB import MongoDB
from cm4.configuration.config import Config
from cm4.vm.Vm import Vm
config = Config()
db = MongoDB(config.get('data.mongo.MONGO_DBNAME'),
config.get('data.mongo.MONGO_USERNAME'),
config.get('data.mongo.MONGO_PASSWORD'),
config.get('data.mongo.MONGO_PORT'))
db.connect_db()
def vm_list():
cloud = request.args.get('cloud')
if cloud:
rep = Vm(cloud).list()
return 'No node is found on {}!\n'.format(cloud) if not rep else \
jsonify(**{'records': [db.var_to_json(x.__dict__) for x in rep]})
else:
return jsonify(**{'records': [db.var_to_json(x) for x in db.db['cloud'].find()]})
def setup(self):
self._conf = Config()
def setup(self):
self.config = Config()
self.azure = Vm('azure')
self.test_node_name = 'cm-test-vm-1'
self.test_node_id = ''
def __init__(self):
self.config = Config().data["cloudmesh"]
def process_arguments(arguments):
"""
Process command line arguments to execute VM actions.
Called from cm4.command.command
:param arguments:
"""
result = None
if arguments.get("--debug"):
pp = pprint.PrettyPrinter(indent=4)
print("vm processing arguments")
pp.pprint(arguments)
default_cloud = Config().data["cloudmesh"]["default"]["cloud"]
vm = Vm(default_cloud)
if arguments.get("list"):
result = vm.list()
elif arguments.get("create"):
# TODO: Reconcile `create` behavior here and in docopts where
# create is called with a `VMCOUNT`.
vm_name = arguments.get("VMNAME")
if vm_name is None:
vm_name = vm.new_name()
vm.create(vm_name)
result = f"Created {vm_name}"
elif arguments.get("start"):
result = vm.start(arguments.get("--vms"))
elif arguments.get("stop"):
result = vm.stop(arguments.get("--vms"))
elif arguments.get("destroy"):
result = vm.destroy(arguments.get("--vms"))
elif arguments.get("status"):
result = vm.status(arguments.get("--vms"))
elif arguments.get("publicip"):
result = vm.get_public_ips(arguments.get('--vms'))
elif arguments.get("ssh"):
# TODO
pass
elif arguments.get("run"):
# TODO
pass
elif arguments.get("script"):
# TODO
pass
return result
def __init__(self):
self.config = Config()
pprint(self.config.dict())