def entry(cls, name):
banner("Register {}".format(name))
name = str(name)
etc_config = ConfigDict("cloudmesh.yaml", etc=True)
config = ConfigDict("cloudmesh.yaml")
clouds = config["cloudmesh.clouds"]
clusters = config["cloudmesh.hpc.clusters"]
if name in clouds:
name = "cloudmesh.clouds.{}.credentials".format(name)
elif name in clusters:
name = "cloudmesh.hpc.clusters.{}.credentials".format(name)
elif not name.startswith("cloudmesh."):
name = "cloudmesh." + name
try:
etc = etc_config[name]
yaml = config[name]
# walk yaml
for key in etc:
if etc[key] == "TBD":
result = input("Enter {:} ({:}): ".format(key, yaml[key]))
if result != '':
yaml[key] = result
config.save()
except Exception as e:
Console.error("Could not find {} in the yaml file".format(name),
traceflag=False)
def get_from_yaml(cls, filename=None, load_order=None, store=True):
"""
:param filename: name of the yaml file
:return: a SSHKeyManager (dict of keys)
"""
config = None
if filename is None:
# default = Config.path_expand(os.path.join("~", ".cloudmesh", "cloudmesh.yaml"))
# config = ConfigDict("cloudmesh.yaml")
filename = "cloudmesh.yaml"
config = ConfigDict(filename)
elif load_order:
config = ConfigDict(filename, load_order)
else:
Console.error("Wrong arguments")
return
config_keys = config["cloudmesh"]["keys"]
default = config_keys["default"]
keylist = config_keys["keylist"]
uri = Config.path_expand(os.path.join("~", ".cloudmesh", filename))
d = []
for key in list(keylist.keys()):
keyname = key
value = keylist[key]
if os.path.isfile(Config.path_expand(value)):
path = Config.path_expand(value)
if store:
Key.add_from_path(path, keyname)
else:
d.append(Key.add_from_path(path, keyname, store=False))
else:
keytype, string, comment = SSHkey._parse(value)
thekey = {
'uri': 'yaml://{}'.format(uri),
'string': value,
'fingerprint': SSHkey._fingerprint(value),
'name': keyname,
'comment': comment,
'source': 'git',
'kind': 'key'
}
thekey["type"], thekey["key"], thekey[
"comment"] = SSHkey._parse(value)
if thekey["comment"] is None:
thekey["comment"] = keyname
if store:
try:
cls.cm.add(thekey)
except:
Console.error("Key already in db", traceflag=False)
else:
d.append(thekey)
if not store:
return d
"""
def entry(cls, name):
banner("Register {}".format(name))
name = str(name)
etc_config = ConfigDict("cloudmesh.yaml", etc=True)
config = ConfigDict("cloudmesh.yaml")
clouds = config["cloudmesh.clouds"]
clusters = config["cloudmesh.hpc.clusters"]
if name in clouds:
name = "cloudmesh.clouds.{}.credentials".format(name)
elif name in clusters:
name = "cloudmesh.hpc.clusters.{}.credentials".format(name)
elif not name.startswith("cloudmesh."):
name = "cloudmesh." + name
try:
etc = etc_config[name]
yaml = config[name]
# walk yaml
for key in etc:
if etc[key] == "TBD":
result = input("Enter {:} ({:}): ".format(key, yaml[key]))
if result != '':
yaml[key] = result
config.save()
except Exception as e:
Console.error("Could not find {} in the yaml file".format(name), traceflag=False)
def ec2(cls, cloud, zipfile):
def sanitize(name):
return name.replace(".zip", "").replace("@", "_")
def find_exports(filename):
with open(filename, "r") as f:
content = f.read()
data = {}
for line in content.split("\n"):
if line.startswith("export "):
line = line.replace("export ", "")
attribute, value = line.split("=", 1)
value = value.replace("${NOVA_KEY_DIR}/", "")
# remove comments
data[attribute] = value.split("#")[0].strip()
return data
base = sanitize(os.path.basename(zipfile))
dest = sanitize(os.path.join(
path_expand("~"),
".cloudmesh",
"clouds",
cloud,
os.path.basename(zipfile)))
Console.msg("Unzip file {} -> {}".format(zipfile, dest))
r = Shell.unzip(zipfile, dest)
rcfile = os.path.join(dest, "ec2rc.sh")
data = find_exports(rcfile)
data["DEST"] = dest
data["CLOUD"] = cloud
d = {
"cm_heading": "{CLOUD}, EC2".format(**data),
"cm_host": None,
"cm_label": "{CLOUD}_ec2".format(**data),
"cm_type": "ec2",
"cm_type_version": "ec2",
"credentials": {
"EC2_ACCESS_KEY": "{EC2_ACCESS_KEY}".format(**data),
"EC2_SECRET_KEY": "{EC2_SECRET_KEY}".format(**data),
"keyname": "TBD_not_used",
"userid": "TBD_not_used",
"EC2_URL": "{EC2_URL}".format(**data),
"EC2_USER_ID": "{EC2_USER_ID}",
"EC2_PRIVATE_KEY": "{DEST}/pk.pem".format(**data),
"EC2_CERT": "{DEST}/cert.pem".format(**data),
"NOVA_CERT": "{DEST}/cacert.pem".format(**data),
"EUCALYPTUS_CERT": "{DEST}/cacert.pem".format(**data),
},
"default": {
"flavor": "m1.small",
"image": "None",
}
}
config = ConfigDict("cloudmesh.yaml")
config["cloudmesh"]["clouds"][cloud] = d
config.save()
def from_file(cls, filename):
"""
Replaces the TBD in cloudmesh.yaml with the contents present in FILEPATH's FILE
:param filename:
:return:
"""
if not os.path.isfile(os.path.expanduser(filename)):
Console.error("{} doesn't exist".format(filename))
return
# BUG should use path separator
path, filename = filename.rsplit("/", 1)
# Config file to be read from
from_config_file = ConfigDict(filename, [path])
config = ConfigDict("cloudmesh.yaml")
# Merging profile
profile = config["cloudmesh"]["profile"]
for profile_key in list(profile.keys()):
if profile[profile_key] == "TBD":
profile[profile_key] = \
from_config_file["cloudmesh"]["profile"][profile_key]
config.save()
# Merging clouds
clouds = config["cloudmesh"]["clouds"]
for cloud in list(clouds.keys()):
cloud_element = clouds[cloud]
for key in list(cloud_element.keys()):
if cloud_element[key] == "TBD":
cloud_element[key] = \
from_config_file["cloudmesh"]["clouds"][cloud][key]
config["cloudmesh"]["clouds"][cloud] = cloud_element
credentials = clouds[cloud]["credentials"]
for key in credentials:
if credentials[key] == "TBD":
credentials[key] = \
from_config_file["cloudmesh"]["clouds"][cloud][
"credentials"][key]
config["cloudmesh"]["clouds"][cloud]["credentials"] = credentials
defaults = clouds[cloud]["default"]
for key in defaults:
if defaults[key] == "TBD":
defaults[key] = \
from_config_file["cloudmesh"]["clouds"][cloud][
"default"][
key]
config["cloudmesh"]["clouds"][cloud]["default"] = defaults
config.save()
Console.ok(
"Overwritten the TBD of cloudmesh.yaml with {} contents".format(
filename))
def set_username(cls, username):
"""
Method that sets the username in yaml.
:param username:
:return:
"""
config = ConfigDict("cloudmesh.yaml")
config['cloudmesh']['profile']['user'] = username
config.save()
def set_username(cls, username):
"""
Method that sets the username in yaml.
:param username:
:return:
"""
config = ConfigDict("cloudmesh.yaml")
config['cloudmesh']['profile']['username'] = username
config.save()
def test_002_set(self):
"""testing to set a value in the dict"""
HEADING()
shutil.copy(self.etc_yaml, self.tmp_yaml)
d = ConfigDict("cloudmesh.yaml", load_order=[self.tmp_dir], verbose=True)
d["cloudmesh"]["profile"]["firstname"] = "Gregor"
d.save()
d = ConfigDict("cloudmesh.yaml", load_order=[self.tmp_dir], verbose=True)
assert d["cloudmesh"]["profile"]["firstname"] == "Gregor"
def get_apikey(endpoint):
config = ConfigDict("cloudmesh.yaml")
cometConf = config["cloudmesh.comet"]
defaultUser = cometConf["username"]
user = input("Comet nucleus username [%s]: " \
% defaultUser)
if not user:
user = defaultUser
password = getpass.getpass()
keyurl = "%s/getkey" % cometConf["endpoints"][endpoint]["nucleus_base_url"]
headers = {"ACCEPT": "application/json"}
r = requests.get(keyurl, headers=headers, auth=HTTPBasicAuth(user, password))
if r.status_code == 200:
keyobj = r.json()
api_key = keyobj["key_name"]
api_secret = keyobj["key"]
config = ConfigDict("cloudmesh.yaml")
config.data["cloudmesh"]["comet"]["endpoints"] \
[endpoint]["auth_provider"] = 'apikey'
config.data["cloudmesh"]["comet"]["endpoints"] \
[endpoint]["apikey"]["api_key"] = api_key
config.data["cloudmesh"]["comet"]["endpoints"] \
[endpoint]["apikey"]["api_secret"] = api_secret
config.save()
Console.ok("api key retrieval and set was successful!")
else:
Console.error("Error getting api key. "
"Please check your username/password", traceflag=False)
def ec2(cls, cloud, zipfile):
def sanitize(name):
return name.replace(".zip", "").replace("@", "_")
def find_exports(filename):
with open(filename, "r") as f:
content = f.read()
data = {}
for line in content.split("\n"):
if line.startswith("export "):
line = line.replace("export ", "")
attribute, value = line.split("=", 1)
value = value.replace("${NOVA_KEY_DIR}/", "")
# remove comments
data[attribute] = value.split("#")[0].strip()
return data
base = sanitize(os.path.basename(zipfile))
dest = sanitize(
os.path.join(path_expand("~"), ".cloudmesh", "clouds", cloud,
os.path.basename(zipfile)))
Console.msg("Unzip file {} -> {}".format(zipfile, dest))
r = Shell.unzip(zipfile, dest)
rcfile = os.path.join(dest, "ec2rc.sh")
data = find_exports(rcfile)
data["DEST"] = dest
data["CLOUD"] = cloud
d = {
"cm_heading": "{CLOUD}, EC2".format(**data),
"cm_host": None,
"cm_label": "{CLOUD}_ec2".format(**data),
"cm_type": "ec2",
"cm_type_version": "ec2",
"credentials": {
"EC2_ACCESS_KEY": "{EC2_ACCESS_KEY}".format(**data),
"EC2_SECRET_KEY": "{EC2_SECRET_KEY}".format(**data),
"keyname": "TBD_not_used",
"userid": "TBD_not_used",
"EC2_URL": "{EC2_URL}".format(**data),
"EC2_USER_ID": "{EC2_USER_ID}",
"EC2_PRIVATE_KEY": "{DEST}/pk.pem".format(**data),
"EC2_CERT": "{DEST}/cert.pem".format(**data),
"NOVA_CERT": "{DEST}/cacert.pem".format(**data),
"EUCALYPTUS_CERT": "{DEST}/cacert.pem".format(**data),
},
"default": {
"flavor": "m1.small",
"image": "None",
}
}
from pprint import pprint
config = ConfigDict("cloudmesh.yaml")
config["cloudmesh"]["clouds"][cloud] = d
config.save()
def test_001_read(self):
HEADING("test if cloudmesh.yaml is loaded")
d = ConfigDict("cloudmesh.yaml", verbose=True)
assert d["cloudmesh"]["profile"]["firstname"] != ""
assert len(d["cloudmesh"]["clouds"]) > 0
try:
d = ConfigDict("cloudmesh.yam", verbose=True)
print("the file cloudmesh.yam should not exists")
assert False
except Exception as e:
assert str(e).startswith("Could not find")
def fill_out_form(cls, filename):
"""
edits profile and clouds from cloudmesh.yaml
:param filename:
:type filename: string
:return:
"""
Console.ok("Filling out form")
print(filename)
config = ConfigDict(filename)
#
# edit profile
#
profile = config["cloudmesh"]["profile"]
keys = list(profile.keys())
# TODO: test this and delete this comment
# get input that works in python 2 and 3
# replaced by
# from builtins import input
# input = None
# try:
# input = input
# except NameError:
# pass
for key in keys:
if profile[key] == "TBD":
result = input("Please enter {:}[{:}]:".format(key, profile[key])) or profile[key]
profile[key] = result
config["cloudmesh"]["profile"] = profile
config.save()
# edit clouds
clouds = config["cloudmesh"]["clouds"]
for cloud in list(clouds.keys()):
print("Editing the credentials for cloud", cloud)
credentials = clouds[cloud]["credentials"]
for key in credentials:
if key not in ["OS_VERSION", "OS_AUTH_URL"] and credentials[key] == "TBD":
result = input("Please enter {:}[{:}]:".format(key, credentials[key])) or credentials[key]
credentials[key] = result
config["cloudmesh"]["clouds"][cloud]["credentials"] = credentials
config.save()
def do_color(self, args, arguments):
"""
::
Usage:
color FLAG
Arguments:
FLAG color mode flag ON/OFF
Description:
Global switch for the console color mode.
One can switch the color mode on/off with
cm color mode ON
cm color mode OFF
By default, the color mode is ON
Examples:
color mode ON
color mode OFF
"""
# default mode is ON
color_mode = True
flag = arguments["FLAG"].lower()
if flag in ["on", "true"]:
color_mode = True
Console.set_theme(color=True)
elif flag in ["off", "false"]:
color_mode = False
Console.set_theme(color=False)
else:
Console.error("Invalid Flag")
return
# Update the cloudmesh.yaml file
config = ConfigDict("cloudmesh.yaml")
config["cloudmesh"]["system"]["console_color"] = color_mode
config.save()
Console.color = color_mode
Console.ok("Color {:}".format(str(color_mode)))
return ""
def test_001_read(self):
HEADING("test if cloudmesh.yaml is loaded")
d = ConfigDict("cloudmesh.yaml", verbose=True)
assert d["cloudmesh"]["profile"]["firstname"] != ""
assert len(d["cloudmesh"]["clouds"]) > 0
"""
def get_nova_credentials(kind="yaml", cloud=None):
d = {}
if kind in ["env"]:
d['version'] = '2'
d['username'] = os.environ['OS_USERNAME']
d['api_key'] = os.environ['OS_PASSWORD']
d['auth_url'] = os.environ['OS_AUTH_URL']
d['project_id'] = os.environ['OS_TENANT_NAME']
d['cacert'] = path_expand(os.environ['OS_CACERT'])
elif kind in ["yaml"]:
if cloud is None:
raise Exception("cloud not specified")
config = dict(
ConfigDict(filename="~/.cloudmesh/cloudmesh.yaml")["cloudmesh"]
["clouds"][cloud])
cred = dict(config["credentials"])
d['version'] = '2'
d['username'] = cred['OS_USERNAME']
d['api_key'] = cred['OS_PASSWORD']
d['auth_url'] = cred['OS_AUTH_URL']
d['project_id'] = cred['OS_TENANT_NAME']
if 'OS_CACERT' in cred:
d['cacert'] = path_expand(cred['OS_CACERT'])
else:
raise Exception("unsupported kind: " + kind)
return d
def logon(self, cloudname):
"""
Logs onto a cloud
:param cloudname:
:return:
"""
CloudProviderOpenstackAPI.cloud_pwd[cloudname] = {}
d = ConfigDict("cloudmesh.yaml")
credentials = d["cloudmesh"]["clouds"][cloudname]["credentials"]
os_password = credentials["OS_PASSWORD"]
prompt = "Password for cloud - {}:".format(cloudname)
if os_password.lower() == "readline":
if 'pwd' in CloudProviderOpenstackAPI.cloud_pwd[cloudname]:
os_password = CloudProviderOpenstackAPI.cloud_pwd[cloudname][
"pwd"]
else:
os_password = getpass.getpass(prompt=prompt)
elif os_password.lower() == "env":
if 'pwd' in CloudProviderOpenstackAPI.cloud_pwd[cloudname]:
os_password = CloudProviderOpenstackAPI.cloud_pwd[cloudname][
"pwd"]
else:
os_password = os.environ.get("OS_PASSWORD",
getpass.getpass(prompt=prompt))
#
# TODO: pwd is not standing for passwd
#
CloudProviderOpenstackAPI.cloud_pwd[cloudname]["pwd"] = os_password
CloudProviderOpenstackAPI.cloud_pwd[cloudname]["status"] = "Active"
return
def var_replacer(self, line, c='$'):
vars = self.var_finder(line, c=c)
for v in vars["normal"]:
value = str(Var.get(v))
line = line.replace(c + v, value)
# replace in line the variable $v with value
for v in vars["os"]:
name = v.replace('os.', '')
if name in os.environ:
value = os.environ[name]
line = line.replace(c + v, value)
else:
Console.error("can not find environment variable {}".format(v))
if c + v in line:
value = os.environ(v)
# replace in line the variable $v with value
for v in vars["dot"]:
try:
config = ConfigDict("cloudmesh.yaml")
print(config["cloudmesh.profile"])
value = config[v]
line = line.replace(c + v, value)
except Exception as e:
Console.error(
"can not find variable {} in cloudmesh.yaml".format(value))
return line
def construct_ip_dict(cls, ip_addr, name="kilo"):
try:
d = ConfigDict("cloudmesh.yaml")
cloud_details = d["cloudmesh"]["clouds"][name]
# Handle Openstack Specific Output
if cloud_details["cm_type"] == "openstack":
ipaddr = {}
for network in ip_addr:
index = 0
for ip in ip_addr[network]:
ipaddr[index] = {}
ipaddr[index]["network"] = network
ipaddr[index]["version"] = ip["version"]
ipaddr[index]["addr"] = ip["addr"]
index += 1
return ipaddr
# Handle EC2 Specific Output
if cloud_details["cm_type"] == "ec2":
print("ec2 ip dict yet to be implemented")
TODO.implement()
# Handle Azure Specific Output
if cloud_details["cm_type"] == "azure":
print("azure ip dict yet to be implemented")
TODO.implement()
except Exception as e:
Error.error("error in vm construct dict", traceback=True)
def list(cls, filename, info=False, output='table'):
"""
lists clouds from cloudmesh.yaml file
:param filename:
:type filename: string
:return:
"""
config = ConfigDict("cloudmesh.yaml")
clouds = config["cloudmesh"]["clouds"]
if info:
Console.ok("Cloudmesh configuration file: {}".format(filename))
print("")
d = {}
for i, key in enumerate(clouds.keys()):
d[i] = {
"id":
i,
"cloud":
key,
"iaas":
config["cloudmesh"]["clouds"][key]["cm_type"],
"version":
config["cloudmesh"]["clouds"][key]["cm_type_version"] or "N/A"
}
return Printer.dict_printer(d,
order=['id', 'cloud', 'iaas', 'version'],
output=output)
def upload(cls, cloud=None, group=None):
if cloud is None:
clouds = ConfigDict("cloudmesh.yaml")["cloudmesh"]["active"]
else:
clouds = [cloud]
if group is None:
rules = cls.list(output='dict')
groups = set()
for g in rules:
r = rules[g]
groups.add(r["group"])
groups = list(groups)
else:
groups = [group]
for cloud in clouds:
for g in groups:
cls.delete_all_rules_cloud(cloud, g)
group = cls.get(name=g, cloud=cloud)
group_cloud = cls.get_group_cloud(cloud, g)
if not group_cloud:
cls.add_group_cloud(cloud, g)
rules = cls.list_rules(group=g, output="dict")
if rules:
for ruleid in rules:
rule = rules[ruleid]
rulename = rule["name"]
cls.add_rule_cloud(cloud, g, rulename)
'''
def setup(self):
cloud = "cybera-ec2"
config = ConfigDict("cloudmesh.yaml")
self.credential = config['cloudmesh']['clouds'][cloud]['credentials']
self.clouddefault = config['cloudmesh']['clouds'][cloud]['default']
# pprint(dict(self.credential))
auth_url = self.credential["EC2_URL"]
data = re.match(r'^http[s]?://(.+):([0-9]+)/([a-zA-Z/]*)', auth_url,
re.M | re.I)
host, port, path = data.group(1), data.group(2), data.group(3)
print("host: " + host)
print("port: " + port)
print("path: " + path)
extra_args = {'path': path}
cls = get_driver(Provider.EC2_US_EAST)
self.driver = cls(self.credential['EC2_ACCESS_KEY'],
self.credential['EC2_SECRET_KEY'],
host=host,
port=port,
**extra_args)
print("DRIVER", self.driver)
assert "libcloud.compute.drivers.ec2.EC2NodeDriver object at" in str(
self.driver)
def list(cls, filename, cloud, info=False, output='table'):
"""
lists clouds from cloudmesh.yaml file
:param filename:
:type filename: string
:return:
"""
config = ConfigDict("cloudmesh.yaml")
clouds = config["cloudmesh"]["clouds"]
if info:
Console.ok("Cloudmesh configuration file: {}".format(filename))
print("")
d = {}
for i, key in enumerate(clouds.keys()):
d[i] = {
"id":
i,
"cloud":
key,
"iaas":
config["cloudmesh"]["clouds"][key]["cm_type"],
"version":
config["cloudmesh"]["clouds"][key]["cm_type_version"] or "",
# "active": "*" if key in config["cloudmesh"]["active"] else "",
"active":
config["cloudmesh"]["active"].index(key) +
1 if key in config["cloudmesh"]["active"] else "",
"default":
"*" if key == cloud else ""
}
return Printer.Printer.write(
d,
order=['id', 'default', 'cloud', 'iaas', 'version', 'active'],
output=output)
def set_level(log_level):
"""
sets th eloglevel in the database and the loglevel file from
cloudmesh.yaml
:param log_level: the loglevel
:return:
"""
# TODO: BUG: This seems inconsistent with our use as it mixes db and
# cloudmesh.yaml.
level = log_level.upper()
Default.set(key=LogUtil.LOG_LEVEL_KEY,
value=log_level,
category=LogUtil.category)
# get log level obj
log_level_obj = LogUtil.get_level_obj(log_level)
# Read the ConfigDict
config = ConfigDict("cloudmesh.yaml")
log_file = config["cloudmesh"]["logging"]["file"]
# Set the logger config
logging.basicConfig(format=LogUtil.FORMAT,
level=log_level_obj,
filename=path_expand(log_file))
LOGGER.info("Set log level to: " + log_level)
return "Ok."
def __init__(self, cloudname, user=None, flat=True):
super(CloudProvider, self).__init__(cloudname, user=user)
try:
d = ConfigDict("cloudmesh.yaml")
if not cloudname in d["cloudmesh"]["clouds"]:
raise ValueError("the cloud {} is not defined in the yaml file. failed."
.format(cloudname))
cloud_details = d["cloudmesh"]["clouds"][cloudname]
if cloud_details["cm_type"] == "openstack":
provider = CloudProviderOpenstackAPI(
cloudname,
cloud_details,
flat=flat)
self.provider = provider
self.provider_class = CloudProviderOpenstackAPI
if cloud_details["cm_type"] == "ec2":
provider = CloudProviderLibcloudEC2(
cloudname,
cloud_details,
flat=flat)
self.provider = provider
self.provider_class = CloudProviderLibcloudEC2
if cloud_details["cm_type"] == "azure":
raise ValueError("azure cloud provider yet implemented. failed.")
TODO.implement()
except Exception as e:
Error.traceback(e)
def check_yaml_for_completeness(cls, filename):
"""
outputs how many values has to be fixed in cloudmesh.yaml file
:param filename: the file name
:type filename: string
:return:
"""
if filename is None:
filename = "cloudmesh.yaml"
config = ConfigDict(filename)
content = config.yaml
Console.ok("Checking the yaml file")
count = 0
output = []
for line in content.split("\n"):
if "TBD" in line:
output.append(textwrap.dedent(line))
count += 1
if count > 0:
Console.error("The file has {:} values to be fixed".format(count))
print("")
for line in output:
Console.error(" " + line, prefix=False)
def initialize(self, cloudname, user=None):
"""
reads the details for the initialization from the cloudname defined in the yaml file
Azure cloud requires subscription_id and service management certificate to initialize the provider
:param cloudname:
:param user:
:return:
"""
confd = ConfigDict("cloudmesh.yaml")
cloudcred = confd['cloudmesh']['clouds']['azure']['credentials']
subscription_id = cloudcred['subscriptionid']
certificate_path = cloudcred['managementcertfile']
# DEBUGGING INFO
# pprint("subscriptionid:"+subscription_id)
# pprint("certificate_path:"+certificate_path)
self.provider = ServiceManagementService(subscription_id,
certificate_path)
self.default_image = confd['cloudmesh']['clouds']['azure']['default'][
'image']
self.default_flavor = confd['cloudmesh']['clouds']['azure']['default'][
'flavor']
self.cloud = "azure"
self.cloud_details = confd['cloudmesh']['clouds']['azure']
self.location = confd["cloudmesh"]["clouds"]["azure"]["default"][
"location"]
def logon(cls, username=None, password=None):
config = ConfigDict("cloudmesh.yaml")
cometConf = config["cloudmesh.comet"]
cls.set_endpoint(cometConf["active"])
cls.set_base_uri(cometConf["endpoints"][cls.endpoint]["nucleus_base_url"])
cls.set_api_version(cometConf["endpoints"][cls.endpoint]["api_version"])
cls.set_auth_provider()
# print (cls.endpoint)
# print (cls.base_uri)
# print (cls.api_version)
# print (cls.auth_provider)
ret = False
if "USERPASS" == cls.auth_provider:
# for unit testing only.
if username is None:
username = cometConf["endpoints"][cls.endpoint]["userpass"]["username"]
if username == '' or username == 'TBD':
username = cometConf["username"]
if password is None:
password = cometConf["endpoints"][cls.endpoint]["userpass"]["password"]
if password.lower() == "readline":
password = getpass.getpass()
elif password.lower() == "env":
password = os.environ.get("COMET_PASSWORD", getpass.getpass())
if cls.token is None:
if cls.auth_uri:
if cls.tunnelled:
authuri = "%s/login/" % cls.local_auth_uri
else:
authuri = "%s/login/" % cls.auth_uri
data = {"username": username, "password": password}
r = requests.post(authuri,
data=json.dumps(data),
headers=cls.HEADER,
verify=cls.verify)
try:
cls.token = r.json()["key"]
cls.AUTH_HEADER['Authorization'] = "Token {:}".format(
cls.token)
except:
ret = False
ret = cls.token
else:
ret = cls.token
elif "APIKEY" == cls.auth_provider:
# print ("API KEY based auth goes here")
cls.api_key = cometConf["endpoints"][cls.endpoint]["apikey"]["api_key"]
cls.api_secret = cometConf["endpoints"][cls.endpoint]["apikey"]["api_secret"]
cls.api_auth = HTTPSignatureAuth(secret=cls.api_secret, headers=["nonce", "timestamp"])
#
# api key based auth does not maintain a session
# once values specified, considered as AuthNed.
if cls.api_key and cls.api_secret and cls.api_auth:
ret = True
else:
print("The specified AUTH Provider Not Currently Supported")
pass
return ret
def load(cls, filename):
config = ConfigDict(filename=filename)["cloudmesh"]
clouds = config["clouds"]
# FINDING DEFAULTS FOR CLOUDS
for cloud in clouds:
db = {
"image": cls.get(name="image", category=cloud),
"flavor": cls.get(name="flavor", category=cloud),
}
defaults = clouds[cloud]["default"]
for attribute in ["image", "flavor"]:
value = db[attribute]
if attribute in defaults:
value = db[attribute] or defaults[attribute]
empty = cls.get(name=attribute, category=cloud)
if empty is None:
cls.set(attribute, value, category=cloud)
# FINDING DEFAUlTS FOR KEYS
# keys:
# default: id_rsa
# keylist:
# id_rsa: ~/.ssh/id_rsa.pub
# key_db = SSHKeyDBManager()
name_key = cls.key
#
# SET DEFAULT KEYS
#
if "keys" in config:
keys = config["keys"]
name = keys["default"]
if name in keys["keylist"]:
value = name_key or keys["keylist"][name]
# key_db.add(value, keyname=name)
# Check if the key is already set
exist_key = cls.key
# Set the key only if there is no existing value in the DB.
if exist_key is None:
cls.set_key(name)
else:
if cls.key is not None and cls.user is not None:
pass
elif cls.key is None and cls.user is not None:
cls.key = cls.user
else:
Console.error(
"Please define a key first, e.g.: cm key add --ssh <keyname>",
traceflag=False)
def get_apikey(endpoint):
config = ConfigDict("cloudmesh.yaml")
cometConf = config["cloudmesh.comet"]
defaultUser = cometConf["username"]
user = input("Comet Nucleus Usename [%s]: " \
% defaultUser)
if not user:
user = defaultUser
password = getpass.getpass()
keyurl = "%s/getkey" % cometConf["endpoints"][endpoint]["nucleus_base_url"]
headers = {"ACCEPT": "application/json"}
r = requests.get(keyurl, headers=headers, auth=HTTPBasicAuth(user, password))
if r.status_code == 200:
keyobj = r.json()
api_key = keyobj["key_name"]
api_secret = keyobj["key"]
config = ConfigDict("cloudmesh.yaml")
config.data["cloudmesh"]["comet"]["endpoints"]\
[endpoint]["auth_provider"] = 'apikey'
config.data["cloudmesh"]["comet"]["endpoints"]\
[endpoint]["apikey"]["api_key"] = api_key
config.data["cloudmesh"]["comet"]["endpoints"]\
[endpoint]["apikey"]["api_secret"] = api_secret
config.save()
Console.ok("api key retrieval and set was successful!")
else:
Console.error("Error getting api key. " \
"Please check your username/password", traceflag=False)
def __init__(self):
self.__dict__ = self.__shared_state
if self.initialized is None:
self.filename = Config.path_expand(os.path.join("~", ".cloudmesh", "cloudmesh.db"))
self.create()
self.create_tables()
self.start()
self.user = ConfigDict(filename="cloudmesh.yaml")["cloudmesh.profile.user"]
def __init__(self, context):
cmd.Cmd.__init__(self)
self.command_topics = {}
self.register_topics()
self.context = context
if self.context.debug:
print("init CloudmeshConsole")
self.prompt = 'ghost> '
self.banner = textwrap.dedent("""
+==========================================================+
. _ .-') ('-. .-') _ .
. ( '.( OO )_ _( OO) ( OO) ) .
. .-----. .-'),-----. ,--. ,--.)(,------./ '._ .
. ' .--./ ( OO' .-. '| `.' | | .---'|'--...__) .
. | |('-. / | | | || | | | '--. .--' .
. /_) |OO )\_) | |\| || |'.'| | (| '--. | | .
. || |`-'| \ | | | || | | | | .--' | | .
. (_' '--'\ `' '-' '| | | | | `---. | | .
. `-----' `-----' `--' `--' `------' `--' .
+==========================================================+
Comet Ghost Shell
""")
# KeyCommands.__init__(self, context)
#
# set default cloud and default group if they do not exist
# use the first cloud in cloudmesh.yaml as default
#
value = Default.get('cloud', 'general')
if value is None:
filename = path_expand("~/.cloudmesh/cloudmesh.yaml")
clouds = ConfigDict(filename=filename)["cloudmesh"]["clouds"]
cloud = clouds.keys()[0]
Default.set('cloud', cloud, 'general')
value = Default.get('default', 'general')
if value is None:
Default.set('default', 'default', 'general')
for c in CloudmeshConsole.__bases__[1:]:
# noinspection PyArgumentList
c.__init__(self, context)
def save():
"""
save the loglevel for a cloud to the cloudmesh.yaml file
"""
# TODO: BUG: this seems inconsistant as loglevels via default
# can now be defined for clouds, but the yaml file only
# specifies one value for all clouds.
# get the log level from database
log_level = Default.get(
name=LogUtil.LOG_LEVEL_KEY,
category=LogUtil.category) or LogUtil.DEFAULT_LOG_LEVEL
# Update the cloudmesh config
config = ConfigDict("cloudmesh.yaml")
config["cloudmesh"]["logging"]["level"] = log_level
# Save this into cloudmesh yaml
config.save()
def set_auth_provider(cls, auth_provider=None):
# try to load from yaml file if not specified
if not auth_provider:
config = ConfigDict("cloudmesh.yaml")
cometConf = config["cloudmesh.comet"]
auth_provider = cometConf["endpoints"][cls.endpoint]["auth_provider"].upper()
# value not set in yaml file, use USERPASS as default
if not auth_provider:
auth_provider = "USERPASS"
cls.auth_provider = auth_provider
def setup(self):
config = ConfigDict("cloudmesh.yaml")
self.data = {
"cloud": "kilo",
}
self.data["tenant"] = config["cloudmesh.clouds"][
self.data["cloud"]]["credentials"]["OS_TENANT_NAME"]
pass
def do_var(self, arg, arguments):
"""
Usage:
var list
var delete NAMES
var NAME=VALUE
var NAME
Arguments:
NAME Name of the variable
NAMES Names of the variable separated by spaces
VALUE VALUE to be assigned
special vars date and time are defined
"""
if arguments['list'] or arg == '' or arg is None:
# self._list_variables()
print(Var.list())
return ""
elif arguments['NAME=VALUE'] and "=" in arguments["NAME=VALUE"]:
(variable, value) = arg.split('=', 1)
if value == "time" or value == "now":
value = datetime.datetime.now().strftime("%H:%M:%S")
elif value == "date":
value = datetime.datetime.now().strftime("%Y-%m-%d")
elif value.startswith("default."):
name = value.replace("default.", "")
value = Default.get(name=name, category="general")
elif "." in value:
try:
config = ConfigDict("cloudmesh.yaml")
value = config[value]
except Exception as e:
Console.error(
"can not find variable {} in cloudmesh.yaml".format(
value))
value = None
# self._add_variable(variable, value)
Var.set(variable, value)
return ""
elif arguments['NAME=VALUE'] and "=" not in arguments["NAME=VALUE"]:
try:
v = arguments['NAME=VALUE']
# Console.ok(str(self.variables[v]))
Console.ok(str(Var.get(v)))
except:
Console.error('variable {:} not defined'.format(
arguments['NAME=VALUE']))
elif arg.startswith('delete'):
variable = arg.split(' ')[1]
Var.delete(variable)
# self._delete_variable(variable)
return ""
def cloudmesh_clouds(request):
config = ConfigDict(filename="cloudmesh.yaml")
clouds = config["cloudmesh.clouds"]
active = config["cloudmesh.active"]
default = Default.get_cloud()
data = {}
attributes = [
'cm_label', 'cm_host', 'cm_heading', 'cm_type', 'cm_type_version'
]
for cloud in clouds:
name = {'cloud': cloud}
data[cloud] = {}
for attribute in attributes:
data[cloud][attribute] = clouds[cloud][attribute]
print clouds[cloud]['cm_type']
if clouds[cloud]['cm_type'] == "ec2":
data[cloud]['username'] = clouds[cloud]['credentials']['userid']
elif clouds[cloud]['cm_type'] == "azure":
data[cloud]['username'] = '******'
elif clouds[cloud]['cm_type'] == "openstack":
data[cloud]['username'] = clouds[cloud]['credentials'][
'OS_USERNAME']
if cloud in active:
data[cloud]['active'] = 'yes'
else:
data[cloud]['active'] = 'no'
if cloud in [default]:
data[cloud]['default'] = 'yes'
else:
data[cloud]['default'] = 'no'
data[cloud]['info'] = ", ".join([
url('d', '/cm/cloud/{cloud}/'.format(**name)),
url('i', '/cm/image/{cloud}/'.format(**name)),
url('f', '/cm/flavor/{cloud}/'.format(**name)),
url('v', '/cm/vm/{cloud}/'.format(**name))
])
order = [
'default', 'active', 'cm_label', 'info', 'username', 'cm_host',
'cm_heading', 'cm_type', 'cm_type_version'
]
header = [
'Default', 'Active', 'Label', 'Info', 'Username', 'Host',
'Description', 'Type', 'Version'
]
pprint(data)
context = {
'data': data,
'title': "Cloud List",
'order': order,
'header': header,
}
return render(request, 'cloudmesh_portal/dict_table.jinja', context)
def from_file(cls, filename):
"""
Replaces the TBD in cloudmesh.yaml with the contents present in FILEPATH's FILE
:param filename:
:return:
"""
if not os.path.isfile(os.path.expanduser(filename)):
Console.error("{} doesn't exist".format(filename))
return
# BUG should use path separator
path, filename = filename.rsplit("/", 1)
# Config file to be read from
from_config_file = ConfigDict(filename, [path])
config = ConfigDict("cloudmesh.yaml")
# Merging profile
profile = config["cloudmesh"]["profile"]
for profile_key in list(profile.keys()):
if profile[profile_key] == "TBD":
profile[profile_key] = \
from_config_file["cloudmesh"]["profile"][profile_key]
config.save()
# Merging clouds
clouds = config["cloudmesh"]["clouds"]
for cloud in list(clouds.keys()):
cloud_element = clouds[cloud]
for key in list(cloud_element.keys()):
if cloud_element[key] == "TBD":
cloud_element[key] = \
from_config_file["cloudmesh"]["clouds"][cloud][key]
config["cloudmesh"]["clouds"][cloud] = cloud_element
credentials = clouds[cloud]["credentials"]
for key in credentials:
if credentials[key] == "TBD":
credentials[key] = \
from_config_file["cloudmesh"]["clouds"][cloud][
"credentials"][key]
config["cloudmesh"]["clouds"][cloud]["credentials"] = credentials
defaults = clouds[cloud]["default"]
for key in defaults:
if defaults[key] == "TBD":
defaults[key] = \
from_config_file["cloudmesh"]["clouds"][cloud][
"default"][
key]
config["cloudmesh"]["clouds"][cloud]["default"] = defaults
config.save()
Console.ok(
"Overwritten the TBD of cloudmesh.yaml with {} contents".format(
filename))
def test_004_yaml(self):
HEADING("test if yaml is produced")
d = ConfigDict("cloudmesh.yaml", verbose=True)
result = d.yaml
try:
assert result.startswith("meta")
except Exception as e:
print("not valid yaml file.")
assert False
def __init__(self, context):
"""
Init Method
:param context:
:return:
"""
self.context = context
if self.context.debug:
print("Init ConfigEnv")
ConfigEnv.yaml_data = ConfigDict("cloudmesh.yaml")
ConfigEnv.env_config_data = {"OS_AUTH_URL": None}
def read_rc_file(cls, host, openrc, force=False):
"""
:param host: the host name
:type host: string
:param openrc: the file name
:type openrc: string
:return:
"""
openrc = Config.path_expand(openrc)
# check if file exists
if not os.path.isfile(openrc):
Console.error("File not found.")
return
with open(openrc, 'r') as f:
lines = f.read().split("\n")
config = ConfigDict("cloudmesh.yaml")
credentials = {}
for line in lines:
if line.strip().startswith("export "):
line = line.replace("export ", "")
key, value = line.split("=", 1)
credentials[key] = value
if host not in config["cloudmesh.clouds"] or force:
config["cloudmesh"]["clouds"][host] = {
"credentials": credentials,
"cm_heading": "TBD",
"cm_host": "TBD",
"cm_label": "TBD",
"cm_type": "TBD",
"cm_type_version": "TBD"}
config.save()
return config["cloudmesh"]["clouds"][host]["credentials"]
def get_apikey():
user = raw_input("Comet Nucleus Usename: ")
password = getpass.getpass()
keyurl = "https://comet-nucleus.sdsc.edu/nucleus/getkey"
headers = {"ACCEPT": "application/json"}
r = requests.get(keyurl, headers=headers, auth=HTTPBasicAuth(user, password))
if r.status_code == 200:
keyobj = r.json()
api_key = keyobj["key_name"]
api_secret = keyobj["key"]
config = ConfigDict("cloudmesh.yaml")
config.data["cloudmesh"]["comet"]["auth_provider"] = 'apikey'
config.data["cloudmesh"]["comet"]["apikey"]["api_key"] = api_key
config.data["cloudmesh"]["comet"]["apikey"]["api_secret"] = api_secret
config.save()
Console.ok("api key retrieval and set was successful!")
else:
Console.error("Error getting api key. " \
"Please check your username/password", traceflag=False)
def do_comet(self, args, arguments):
"""
::
Usage:
comet init
comet active [ENDPOINT]
comet ll [CLUSTERID] [--format=FORMAT] [--endpoint=ENDPOINT]
comet cluster [--concise|--status] [CLUSTERID]
[--format=FORMAT]
[--sort=SORTKEY]
[--endpoint=ENDPOINT]
comet computeset [COMPUTESETID]
[--allocation=ALLOCATION]
[--cluster=CLUSTERID]
[--state=COMPUTESESTATE]
[--endpoint=ENDPOINT]
comet start CLUSTERID [--count=NUMNODES] [COMPUTENODEIDS]
[--allocation=ALLOCATION]
[--reservation=RESERVATION]
[--walltime=WALLTIME]
[--endpoint=ENDPOINT]
comet terminate COMPUTESETID [--endpoint=ENDPOINT]
comet power (on|off|reboot|reset|shutdown) CLUSTERID [NODESPARAM]
[--endpoint=ENDPOINT]
comet console [--link] CLUSTERID [COMPUTENODEID]
[--endpoint=ENDPOINT]
comet node info CLUSTERID [COMPUTENODEID] [--format=FORMAT]
[--endpoint=ENDPOINT]
comet node rename CLUSTERID OLDNAMES NEWNAMES
[--endpoint=ENDPOINT]
comet iso list [--endpoint=ENDPOINT]
comet iso upload [--isoname=ISONAME] PATHISOFILE
[--endpoint=ENDPOINT]
comet iso attach ISOIDNAME CLUSTERID [COMPUTENODEIDS]
[--endpoint=ENDPOINT]
comet iso detach CLUSTERID [COMPUTENODEIDS]
[--endpoint=ENDPOINT]
comet reservation (list|create|update|delete)
Options:
--endpoint=ENDPOINT Specify the comet nucleus service
endpoint to work with, e.g., dev
or production
--format=FORMAT Format is either table, json, yaml,
csv, rest
[default: table]
--sort=SORTKEY Sorting key for the table view
--count=NUMNODES Number of nodes to be powered on.
When this option is used, the comet system
will find a NUMNODES number of arbitrary nodes
that are available to boot as a computeset
--allocation=ALLOCATION Allocation to charge when power on
node(s)
--reservation=RESERVATION Submit the request to an existing reservation
--walltime=WALLTIME Walltime requested for the node(s).
Walltime could be an integer value followed
by a unit (m, h, d, w, for minute, hour, day,
and week, respectively). E.g., 3h, 2d
--isoname=ISONAME Name of the iso image after being stored remotely.
If not specified, use the original filename
--state=COMPUTESESTATE List only computeset with the specified state.
The state could be submitted, running, completed
--link Whether to open the console url or just show the link
--concise Concise table view for cluster info
--status Cluster table view displays only those columns showing state of nodes
Arguments:
ENDPOINT Service endpoint based on the yaml config file.
By default it's either dev or production.
CLUSTERID The assigned name of a cluster, e.g. vc1
COMPUTESETID An integer identifier assigned to a computeset
COMPUTENODEID A compute node name, e.g., vm-vc1-0
If not provided, the requested action will be taken
on the frontend node of the specified cluster
COMPUTENODEIDS A set of compute node names in hostlist format,
e.g., vm-vc1-[0-3]
One single node is also acceptable: vm-vc1-0
If not provided, the requested action will be taken
on the frontend node of the specified cluster
NODESPARAM Specifying the node/nodes/computeset to act on.
In case of integer, will be intepreted as a computesetid;
in case of a hostlist format, e.g., vm-vc1-[0-3], a group
of nodes; or a single host is also acceptable,
e.g., vm-vc1-0
ISONAME Name of an iso image at remote server
ISOIDNAME Index or name of an iso image at the remote server.
The index is based on the list from 'comet iso list'.
PATHISOFILE The full path to the iso image file to be uploaded
OLDNAMES The list of current node names to be renamed, in hostlist
format. A single host is also acceptable.
NEWNAMES The list of new names to rename to, in hostlist format.
A single host is also acceptable.
"""
# back up of all the proposed commands/options
"""
comet status
comet tunnel start
comet tunnel stop
comet tunnel status
comet logon
comet logoff
comet ll [CLUSTERID] [--format=FORMAT]
comet docs
comet info [--user=USER]
[--project=PROJECT]
[--format=FORMAT]
comet cluster [CLUSTERID][--name=NAMES]
[--user=USER]
[--project=PROJECT]
[--hosts=HOSTS]
[--start=TIME_START]
[--end=TIME_END]
[--hosts=HOSTS]
[--format=FORMAT]
comet computeset [COMPUTESETID]
comet start ID
comet stop ID
comet power on CLUSTERID [NODESPARAM]
[--allocation=ALLOCATION]
[--walltime=WALLTIME]
comet power (off|reboot|reset|shutdown) CLUSTERID [NODESPARAM]
comet console CLUSTERID [COMPUTENODEID]
comet delete [all]
[--user=USER]
[--project=PROJECT]
[--name=NAMES]
[--hosts=HOSTS]
[--start=TIME_START]
[--end=TIME_END]
[--host=HOST]
comet delete --file=FILE
comet update [--name=NAMES]
[--hosts=HOSTS]
[--start=TIME_START]
[--end=TIME_END]
comet add [--user=USER]
[--project=PROJECT]
[--host=HOST]
[--description=DESCRIPTION]
[--start=TIME_START]
[--end=TIME_END]
NAME
comet add --file=FILENAME
Options:
--user=USER user name
--name=NAMES Names of the vcluster
--start=TIME_START Start time of the vcluster, in
YYYY/MM/DD HH:MM:SS format.
[default: 1901-01-01]
--end=TIME_END End time of the vcluster, in YYYY/MM/DD
HH:MM:SS format. In addition a duratio
can be specified if the + sign is the
first sig The duration will than be
added to the start time.
[default: 2100-12-31]
--project=PROJECT project id
--host=HOST host name
--description=DESCRIPTION description summary of the vcluster
--file=FILE Adding multiple vclusters from one file
--format=FORMAT Format is either table, json, yaml,
csv, rest
[default: table]
--allocation=ALLOCATION Allocation to charge when power on
node(s)
--walltime=WALLTIME Walltime requested for the node(s)
Arguments:
FILENAME the file to open in the cwd if . is
specified. If file in in cwd
you must specify it with ./FILENAME
Opens the given URL in a browser window.
"""
"""
if not arguments["tunnel"] and Comet.tunnelled and not Comet.is_tunnel():
Console.error("Please establish a tunnel first with:")
print
print (" comet tunnel start")
print
return ""
try:
if not arguments["tunnel"]:
logon = Comet.logon()
if logon is False:
Console.error("Could not logon")
return ""
except:
Console.error("Could not logon")
# pprint (arguments)
output_format = arguments["--format"] or "table"
if arguments["status"]:
Comet.state()
elif arguments["tunnel"] and arguments["start"]:
Comet.tunnel(True)
elif arguments["tunnel"] and arguments["stop"]:
Comet.tunnel(False)
elif arguments["tunnel"] and arguments["status"]:
Comet.state()
elif arguments["logon"]:
if self.context.comet_token is None:
if Comet.logon():
Console.ok("logging on")
self.context.comet_token = Comet.token
else:
Console.error("could not logon")
else:
Console.error("already logged on")
elif arguments["logoff"]:
if self.context.comet_token is None:
Console.error("not logged in")
else:
if Comet.logoff():
Console.ok("Logging off")
self.context.comet_token = None
else:
Console.error(
"some issue while logging off. Maybe comet not reachable")
elif arguments["docs"]:
Comet.docs()
elif arguments["info"]:
Console.error("not yet implemented")
elif arguments["add"]:
print ("add the cluster")
elif arguments["start"]:
cluster_id = arguments["ID"]
print("start", cluster_id)
Cluster.start(cluster_id)
elif arguments["stop"]:
cluster_id = arguments["ID"]
print("stop", cluster_id)
Cluster.stop(cluster_id)
elif arguments["ll"]:
"""
if arguments["init"]:
print ("Initializing the comet configuration file...")
config = ConfigDict("cloudmesh.yaml")
# for unit testing only.
cometConf = config["cloudmesh.comet"]
endpoints = []
# print (cometConf.keys())
if "endpoints" in cometConf.keys():
endpoints = cometConf["endpoints"].keys()
if len(endpoints) < 1:
Console.error("No service endpoints available. "
"Please check the config template",
traceflag=False)
return ""
if "username" in cometConf.keys():
default_username = cometConf['username']
# print (default_username)
if 'TBD' == default_username:
set_default_user = \
input("Set a default username (RETURN to skip): ")
if set_default_user:
config.data["cloudmesh"]["comet"]["username"] = \
set_default_user
config.save()
Console.ok("Comet default username set!")
if "active" in cometConf.keys():
active_endpoint = cometConf['active']
set_active_endpoint = \
input("Set the active service endpoint to use. "
"The availalbe endpoints are - %s [%s]: "
% ("/".join(endpoints),
active_endpoint)
)
if set_active_endpoint:
if set_active_endpoint in endpoints:
config.data["cloudmesh"]["comet"]["active"] = \
set_active_endpoint
config.save()
Console.ok("Comet active service endpoint set!")
else:
Console.error("The provided endpoint does not match "
"any available service endpoints. Try %s"
% "/".join(endpoints),
traceflag=False)
if cometConf['active'] in endpoints:
endpoint_url = cometConf["endpoints"] \
[cometConf['active']]["nucleus_base_url"]
api_version = cometConf["endpoints"] \
[cometConf['active']]["api_version"]
set_endpoint_url = \
input("Set the base url for the nucleus %s service [%s]: " \
% (cometConf['active'],
endpoint_url)
)
if set_endpoint_url:
if set_endpoint_url != endpoint_url:
config.data["cloudmesh"]["comet"]["endpoints"] \
[cometConf['active']]["nucleus_base_url"] \
= set_endpoint_url
config.save()
Console.ok("Service base url set!")
set_api_version = \
input("Set the api version for the nucleus %s service [%s]: " \
% (cometConf['active'],
api_version)
)
if set_api_version:
if set_api_version != api_version:
config.data["cloudmesh"]["comet"]["endpoints"] \
[cometConf['active']]["api_version"] \
= set_api_version
config.save()
Console.ok("Service api version set!")
print("Authenticating to the nucleus %s " \
"service and obtaining the apikey..." \
% cometConf['active'])
Comet.get_apikey(cometConf['active'])
return ''
# Comet.get_apikey()
if arguments["active"]:
config = ConfigDict("cloudmesh.yaml")
cometConf = config["cloudmesh.comet"]
endpoint = arguments["ENDPOINT"] or None
# parameter specified, intended to change
if endpoint:
if "endpoints" in cometConf.keys():
endpoints = cometConf["endpoints"].keys()
if endpoint in endpoints:
config.data["cloudmesh"] \
["comet"] \
["active"] = endpoint
config.save()
Console.ok("Comet active service endpoint set"
" to: %s" % endpoint)
else:
Console.error("The provided endpoint does not match "
"any available service endpoints. Try %s."
% "/".join(endpoints),
traceflag = False)
else:
Console.error("No available endpoint to set. "
"Check config file!",
traceflag=False)
else:
if "active" in cometConf.keys():
active_endpoint = cometConf['active']
Console.ok("Current active service endpoint is: %s"
% active_endpoint)
else:
Console.error("Cannot set active endpoint. "
"Check config file!",
traceflag = False)
try:
endpoint = None
config = ConfigDict("cloudmesh.yaml")
cometConf = config["cloudmesh.comet"]
if arguments["--endpoint"]:
endpoint = arguments["--endpoint"]
if "endpoints" in cometConf.keys():
endpoints = cometConf["endpoints"].keys()
if endpoint not in endpoints:
Console.error("The provided endpoint does not match "
"any available service endpoints. Try %s."
% "/".join(endpoints),
traceflag = False)
return ''
logon = Comet.logon(endpoint=endpoint)
if logon is False:
Console.error("Could not logon. Please try first:\n"
"cm comet init",
traceflag = False)
return ""
except:
Console.error("Could not logon",
traceflag = False)
output_format = arguments["--format"] or "table"
if arguments["ll"]:
cluster_id = arguments["CLUSTERID"] or None
print(Cluster.simple_list(cluster_id, format=output_format))
elif arguments["cluster"]:
view = "FULL"
if arguments["--concise"]:
view = "CONCISE"
if arguments["--status"]:
view = "STATE"
cluster_id = arguments["CLUSTERID"]
sortkey = arguments["--sort"]
print(Cluster.list(cluster_id,
format=output_format,
sort=sortkey,
view=view))
elif arguments["computeset"]:
computeset_id = arguments["COMPUTESETID"] or None
cluster = arguments["--cluster"] or None
state = arguments["--state"] or None
allocation = arguments["--allocation"] or None
cluster = arguments["--cluster"] or None
print (Cluster.computeset(computeset_id, cluster, state, allocation))
elif arguments["start"]:
clusterid = arguments["CLUSTERID"]
numnodes = arguments["--count"] or None
computenodeids = arguments["COMPUTENODEIDS"] or None
# check allocation information for the cluster
cluster = Cluster.list(clusterid, format='rest')
try:
allocations = cluster[0]['allocations']
except:
# print (cluster)
Console.error("No allocation available for the specified cluster."\
"Please check with the comet help team",
traceflag=False)
return ""
# checking whether the computesetids is in valid hostlist format
if computenodeids:
try:
hosts_param = hostlist.expand_hostlist(computenodeids)
except hostlist.BadHostlist:
Console.error("Invalid hosts list specified!",
traceflag=False)
return ""
elif numnodes:
try:
param = int(numnodes)
except ValueError:
Console.error("Invalid count value specified!",
traceflag=False)
return ""
if param <= 0:
Console.error("count value has to be greather than zero",
traceflag=False)
return ""
numnodes = param
else:
Console.error("You have to specify either the count of nodes, " \
"or the names of nodes in hostlist format",
traceflag=False)
return ""
walltime = arguments["--walltime"] or None
allocation = arguments["--allocation"] or None
reservation = arguments["--reservation"] or None
# validating walltime and allocation parameters
walltime = Cluster.convert_to_mins(walltime)
if not walltime:
print("No valid walltime specified. " \
"Using system default (2 days)")
if not allocation:
if len(allocations) == 1:
allocation = allocations[0]
else:
allocation = Cluster.display_get_allocation(allocations)
# issuing call to start a computeset with specified parameters
print(Cluster.computeset_start(clusterid,
computenodeids,
numnodes,
allocation,
reservation,
walltime)
)
elif arguments["terminate"]:
computesetid = arguments["COMPUTESETID"]
print(Cluster.computeset_terminate(computesetid))
elif arguments["power"]:
clusterid = arguments["CLUSTERID"] or None
fuzzyparam = arguments["NODESPARAM"] or None
# parsing nodesparam for proper action
if fuzzyparam:
try:
param = int(fuzzyparam)
subject = 'COMPUTESET'
except ValueError:
param = fuzzyparam
try:
hosts_param = hostlist.expand_hostlist(fuzzyparam)
subject = 'HOSTS'
except hostlist.BadHostlist:
Console.error("Invalid hosts list specified!",
traceflag=False)
return ""
else:
subject = 'FE'
param = None
if arguments["on"]:
action = "on"
elif arguments["off"]:
action = "off"
elif arguments["reboot"]:
action = "reboot"
elif arguments["reset"]:
action = "reset"
elif arguments["shutdown"]:
action = "shutdown"
else:
action = None
print (Cluster.power(clusterid,
subject,
param,
action)
)
elif arguments["console"]:
clusterid = arguments["CLUSTERID"]
linkonly = False
if arguments["--link"]:
linkonly = True
nodeid = None
if 'COMPUTENODEID' in arguments:
nodeid = arguments["COMPUTENODEID"]
Comet.console(clusterid, nodeid, linkonly)
elif arguments["iso"]:
if arguments["list"]:
isos = (Comet.list_iso())
idx = 0
for iso in isos:
if iso.startswith("public/"):
iso = iso.split("/")[1]
idx += 1
print ("{}: {}".format(idx, iso))
if arguments["upload"]:
isofile = arguments["PATHISOFILE"]
isofile = os.path.abspath(isofile)
if os.path.isfile(isofile):
if arguments["--isoname"]:
filename = arguments["--isoname"]
else:
filename = os.path.basename(isofile)
else:
print ("File does not exist - {}" \
.format(arguments["PATHISOFILE"]))
return ""
print(Comet.upload_iso(filename, isofile))
elif arguments["attach"]:
isoidname = arguments["ISOIDNAME"]
clusterid = arguments["CLUSTERID"]
computenodeids = arguments["COMPUTENODEIDS"] or None
print(Cluster.attach_iso(isoidname, clusterid, computenodeids))
elif arguments["detach"]:
clusterid = arguments["CLUSTERID"]
computenodeids = arguments["COMPUTENODEIDS"] or None
print(Cluster.detach_iso(clusterid, computenodeids))
elif arguments["node"]:
if arguments["info"]:
clusterid = arguments["CLUSTERID"]
nodeid = arguments["COMPUTENODEID"]
print (Cluster.node_info(clusterid, nodeid=nodeid, format=output_format))
elif arguments["rename"]:
clusterid = arguments["CLUSTERID"]
oldnames = Parameter.expand(arguments["OLDNAMES"])
newnames = Parameter.expand(arguments["NEWNAMES"])
if len(oldnames) != len(newnames):
Console.error("Length of OLDNAMES and NEWNAMES have to be the same",
traceflag=False)
return ""
else:
for newname in newnames:
if newname.strip() == "":
Console.error("Newname cannot be empty string",
traceflag=False)
return ""
cluster_data = Cluster.list(clusterid, format="rest")
if len(cluster_data) > 0:
computes = cluster_data[0]["computes"]
nodenames = [x["name"] for x in computes]
else:
Console.error("Error obtaining the cluster information",
traceflag=False)
return ""
# check if new names ar not already taken
# to be implemented
# print (oldnames)
# print (newnames)
# print (nodenames)
oldset = set(oldnames)
newset = set(newnames)
currentset = set(nodenames)
# at least one OLDNAME does not exist
if not oldset <= currentset:
Console.error("Not all OLDNAMES are valid", traceflag=False)
return ""
else:
# those unchanged nodes
keptset = currentset - oldset
# duplication between name of unchanged nodes and
# the requested NEWNAMES
if keptset & newset != set():
Console.error("Not proceeding as otherwise introducing "\
"duplicated names",
traceflag=False)
else:
for i in range(0,len(oldnames)):
oldname = oldnames[i]
newname = newnames[i]
print ("%s -> %s" % (oldname, newname))
confirm = input("Confirm batch renaming (Y/y to confirm, "\
"any other key to abort):")
if confirm.lower() == 'y':
print ("Conducting batch renaming")
for i in range(0,len(oldnames)):
oldname = oldnames[i]
newname = newnames[i]
print (Cluster.rename_node(clusterid,
oldname,
newname))
else:
print ("Action aborted!")
elif arguments["reservation"]:
if arguments["create"] or \
arguments["update"] or \
arguments["delete"]:
Console.info("Operation not supported. Please contact XSEDE helpdesk for help!")
if arguments["list"]:
if "hpcinfo" in cometConf:
hpcinfourl = cometConf["hpcinfo"]["endpoint"]
else:
Console.error("Admin feature not configured for this client", traceflag = False)
return ""
ret = requests.get("%s/reservations/%s" % (hpcinfourl,
cometConf['active'])
)
jobs = ret.json()
result = Printer.write(jobs)
print (result)
return ""
def __init__(self, context):
cmd.Cmd.__init__(self)
self.command_topics = {}
self.register_topics()
self.context = context
if self.context.debug:
print("init CloudmeshConsole")
self.prompt = 'cm> '
self.banner = textwrap.dedent("""
+=======================================================+
. ____ _ _ _ .
. / ___| | ___ _ _ __| |_ __ ___ ___ ___| |__ .
. | | | |/ _ \| | | |/ _` | '_ ` _ \ / _ \/ __| '_ \ .
. | |___| | (_) | |_| | (_| | | | | | | __/\__ \ | | | .
. \____|_|\___/ \__,_|\__,_|_| |_| |_|\___||___/_| |_| .
+=======================================================+
Cloudmesh Shell
""")
# KeyCommands.__init__(self, context)
#
# set default cloud and default group if they do not exist
# use the first cloud in cloudmesh.yaml as default
#
filename = path_expand("~/.cloudmesh/cloudmesh.yaml")
create_cloudmesh_yaml(filename)
# sys,exit(1)
value = Default.get('cloud', cloud='general')
if value is None:
clouds = ConfigDict(filename=filename)["cloudmesh"]["clouds"]
cloud = clouds.keys()[0]
Default.set('cloud', cloud, cloud='general')
value = Default.get('default', cloud='general')
if value is None:
Default.set('default', 'default', cloud='general')
cluster = 'india' # hardcode a value if not defined
value = Default.get('cluster', cloud='general')
if value is None:
try:
hosts = ssh_config().names()
if hosts is not None:
cluster = hosts[0]
except:
pass # use the hardcoded cluster
else:
cluster = value
Default.set('cluster', cluster, cloud='general')
#
# Read cloud details from yaml file
#
filename = 'cloudmesh.yaml'
config = ConfigDict(filename=filename)["cloudmesh"]
clouds = config["clouds"]
defaults = {'clouds': {},
'key': {}}
for cloud in clouds:
if "default" in config['clouds'][cloud]:
defaults['clouds'][cloud] = config["clouds"][cloud]['default']
if "default" in config["keys"]:
defaults["keys"] = config["keys"]["default"]
else:
defaults['key'] = None
for cloud in defaults["clouds"]:
for default, value in defaults["clouds"][cloud].iteritems():
Default.set(default, value, cloud=cloud)
for c in CloudmeshConsole.__bases__[1:]:
# noinspection PyArgumentList
c.__init__(self, context)
def do_register(self, args, arguments):
"""
::
Usage:
register info
register backup
register new [--force] [--dryrun]
register clean [--force]
register list ssh [--format=FORMAT]
register list [--yaml=FILENAME][--info][--format=FORMAT]
register cat [--yaml=FILENAME]
register edit [--yaml=FILENAME]
register user [USERNAME]
register cloud [CLOUD] [--force]
register remote [CLOUD] [--force]
register export HOST [--password] [--format=FORMAT]
register source HOST
register merge FILEPATH
register form [--yaml=FILENAME]
register check [--yaml=FILENAME]
register test [--yaml=FILENAME]
register json HOST
register env [--provider=PROVIDER]
register ec2 CLOUD EC2ZIP
register ENTRY
managing the registered clouds in the cloudmesh.yaml file.
It looks for it in the current directory, and than in
~/.cloudmesh. If the file with the cloudmesh.yaml name is
there it will use it. If neither location has one a new
file will be created in ~/.cloudmesh/cloudmesh.yaml. Some
defaults will be provided. However you will still need to
fill it out with valid entries.
Arguments:
HOST the host name
USER the user name
FILEPATH the path of the file
CLOUD the cloud name
PROVIDER the provider or type of cloud [Default: openstack]
USERNAME Username that would be registered in yaml. Defaults to OS username.
Options:
--provider=PROVIDER Provider to be used for cloud. Values are:
openstack, azure, ec2.
--version=VERSION Version of the openstack cloud.
--openrc=OPENRC The location of the openrc file
--password Prints the password
--force ignore interactive questions and execute
the action
Description:
register info
lists the clouds specified in the cloudmesh.yaml
file in the current directory, and then in ~/.cloudmesh
register list [--yaml=FILENAME] [--name] [--info]
lists the clouds specified in the cloudmesh.yaml file. If
info is specified it also prints the location of the yaml
file.
register list ssh
lists hosts from ~/.ssh/config
register cat [--yaml=FILENAME]
outputs the cloudmesh.yaml file
register edit [--yaml=FILENAME]
edits the cloudmesh.yaml file
register export HOST [--format=FORMAT]
prints the contents of an openrc.sh file based on the
information found in the cloudmesh.yaml file.
register remote CLOUD [--force]
reads the Openstack OPENRC file from a remote host that
is described in cloudmesh.yaml file. We assume that
the file has already a template for this host. If
not it can be created from other examples before
you run this command.
It uses the OS_OPENRC variable to locate the file and
copy it onto your computer.
register merge FILENAME
Replaces the TBD in cloudmesh.yaml with the contents
present in the named file
register form [--yaml=FILENAME]
interactively fills out the form wherever we find TBD.
register check [--yaml=FILENAME]
checks the yaml file for completness
register test [--yaml=FILENAME]
checks the yaml file and executes tests to check if
we can use the cloud. TODO: maybe this should be in
a test command
register json host
displays the host details in json format
register remote CLOUD
registers a remote cloud and copies the openrc file
specified in the credentials of the cloudmesh.yaml
register CLOUD --dir
Copies the entire directory from the cloud and puts it in
~/.cloudmesh/clouds/host
For kilo, The directory would be copied to
~/.cloudmesh/clouds/kilo
register env [--provider=PROVIDER] [HOSTNAME]
Reads env OS_* variables and registers a new cloud in yaml,
interactively. Default PROVIDER is openstack and HOSTNAME
is localhost.
register user [USERNAME]
Sets the user in yaml with the value provided.
"""
# from pprint import pprint
# pprint(arguments)
def _get_config_yaml_file(arguments):
filename = arguments["--yaml"] or "cloudmesh.yaml"
filename = Config.find_file(filename)
return filename
def exists(filename):
return os.path.isfile(filename)
def export(host, output):
config = ConfigDict("cloudmesh.yaml")
credentials = dict(
config["cloudmesh"]["clouds"][host]["credentials"])
if not arguments["--password"]:
credentials["OS_PASSWORD"] = "******"
if output is None:
for attribute, value in credentials.items():
print("export {}={}".format(attribute, value))
elif output == "table":
print(Printer.attribute(credentials))
else:
print(Printer.write(credentials, output=output))
# TODO: bug csv does not work
if arguments["info"]:
filename = _get_config_yaml_file(arguments)
if os.path.isfile(filename):
Console.ok("File '{}' exists. ok.".format(filename))
Console.ok("The yaml file contains the following templates:")
d = CloudRegister.list(filename,
Default.cloud,
info=False,
output="table")
print(d)
else:
Console.error("File {} does not exist".format(filename))
return ""
elif arguments["backup"]:
name = backup_name("~/.cloudmesh/cloudmesh.yaml")
configfile = path_expand("~/.cloudmesh/cloudmesh.yaml")
print (name)
try:
copy(configfile, name)
Console.ok("Bakup copy created: {}. ok.".format(name))
except:
Console.error("Could not create a backup copy from {}".format(configfile))
return ""
elif arguments["new"]:
import shutil
import cloudmesh_client.etc
config = ConfigDict("cloudmesh.yaml")
data = dotdict({
'dir': cloudmesh_client.etc.__file__,
'filename': os.path.join(
os.path.dirname(cloudmesh_client.etc.__file__),
"cloudmesh.yaml"),
'yamlfile': path_expand("~/.cloudmesh/cloudmesh.yaml"),
'dryrun': arguments['--dryrun']
})
Console.ok(data.filename)
force = arguments["--force"]
if not force:
force = yn_choice("Would you like create a new configuration file at {}".format(data.yamlfile))
if force:
if not data.dryrun:
config.make_a_copy(location=data.yamlfile)
shutil.copyfile(data.filename, data.yamlfile)
print("copy ")
print("From: ", data.filename)
print("To: ", data.yamlfile)
# filename = _get_config_yaml_file(arguments)
# if _exists(filename):
# Console.ok("File '{}' exists. ok.".format(filename))
# else:
# Console.error("File {} does not exist".format(filename))
return ""
elif arguments["clean"]:
filename = _get_config_yaml_file(arguments)
force = arguments["--force"] or False
if filename is not None:
print(filename, force)
if exists(filename):
print("Delete cloudmesh.yaml file:", filename)
if not force:
force = yn_choice("Would you like to delete the "
"cloudmesh.yaml file")
print(force)
if force:
os.remove(filename)
Console.ok("Deleted the file " + filename + ". ok.")
else:
Console.ok("Please use Y to delete the file.")
pass
else:
Console.error("File {} does not exist".format(filename))
else:
Console.error("No cloudmesh.yaml file found.")
return ""
elif arguments["cat"]:
filename = _get_config_yaml_file(arguments)
if exists(filename):
with open(filename, 'r') as f:
lines = f.read().split("\n")
print('\n'.join(lines))
else:
Console.error("File {} does not exist".format(filename))
return ""
elif arguments["edit"]:
filename = _get_config_yaml_file(arguments)
if exists(filename):
try:
data = {"editor": os.environ["EDITOR"],
"filename": filename}
Console.ok("editing file " + filename)
os.system("{editor} {filename}".format(**data))
except:
Console.error("No operating system environment variable EDITOR set.", traceflag=False)
else:
Console.error("File {} does not exist".format(filename), traceflag=False)
return ""
elif arguments['list'] and arguments['ssh']:
output = arguments['--format'] or 'table'
hosts = CloudRegister.list_ssh()
print(Printer.list(hosts, output=output))
return ""
elif arguments['list']:
filename = _get_config_yaml_file(arguments)
info = arguments["--info"] or False
output = arguments["--format"] or "table"
if not filename:
Console.error("File {} doesn't exist".format(filename))
else:
d = CloudRegister.list(filename,
Default.cloud,
info=info,
output=output)
print(d)
return ""
elif arguments['check']:
filename = _get_config_yaml_file(arguments)
if not filename:
Console.error("File {} doesn't exist".format(
arguments["--yaml"] or 'cloudmesh.yaml'))
else:
CloudRegister.check_yaml_for_completeness(filename)
return ""
elif arguments['merge']:
filename = arguments['FILENAME']
CloudRegister.from_file(filename)
return ""
elif arguments['test']:
filename = _get_config_yaml_file(arguments)
CloudRegister.test(filename)
return ""
elif arguments['form']:
filename = _get_config_yaml_file(arguments)
if not filename:
Console.error("File {} doesn't exist".format(
arguments["--yaml"] or 'cloudmesh.yaml'))
else:
CloudRegister.fill_out_form(filename)
return ""
elif arguments['source']:
host = arguments['HOST']
config = ConfigDict("cloudmesh.yaml")
credentials = dict(
config["cloudmesh"]["clouds"][host]["credentials"])
# unset
variables = list(os.environ)
for attribute in variables:
if attribute.startswith("OS_"):
print("x ", attribute)
del os.environ[attribute]
# set
for attribute, value in credentials.items():
os.putenv(attribute, value)
print("+ ", attribute)
export(host, "table")
return ""
elif arguments['export']:
output = arguments['--format']
host = arguments['HOST']
try:
variables = list(os.environ)
for attribute in variables:
if attribute.startswith("OS_"):
print("unset ", attribute)
del os.environ[attribute]
export(host, output)
except:
Console.error ("The export may not include all values", traceflag=False)
return ""
elif arguments['json']:
host = arguments['HOST']
result = CloudRegister.get(host)
if result:
print(json.dumps(result, indent=4))
else:
print("Cloud {:} is not described in cloudmesh.yaml".format(
host))
return ""
elif arguments['remote']:
force = arguments['--force']
cloud = arguments['CLOUD']
if cloud is None:
# clouds = [ConfigDict(filename="cloudmesh.yaml")["cloudmesh"]["active"][0]]
clouds = ["kilo"] # hardcode to kilo for now
else:
clouds = [cloud]
for cloud in clouds:
CloudRegister.remote(cloud, force)
export(cloud, "table")
config = ConfigDict("cloudmesh.yaml")
if config["cloudmesh.profile.user"] == "TBD":
name = config["cloudmesh.clouds.kilo.credentials.OS_USERNAME"]
config["cloudmesh"]["profile"]["user"] = name
config.save()
return ""
elif arguments['ec2']:
cloud = arguments['CLOUD']
zipfile = arguments['EC2ZIP']
if cloud is None:
clouds = [ConfigDict(filename="cloudmesh.yaml")["cloudmesh"]["active"][0]]
else:
clouds = [cloud]
for cloud in clouds:
CloudRegister.ec2(cloud, zipfile)
export(cloud, "table")
return ""
elif arguments['env']:
try:
CloudRegister.from_environ(arguments['--provider'])
except Exception as e:
Error.traceback(e)
return ""
elif arguments['cloud']:
"""
if arguments['--dir']:
cloud = arguments['--name']
directory = arguments['--dir']
Console.ok(directory)
CloudRegister.directory(cloud, directory)
else:
"""
values_to_replace = ['tbd', 'null', 'tbd_not_used']
cloud = arguments['CLOUD']
if cloud is None:
clouds = [ConfigDict(filename="cloudmesh.yaml")["cloudmesh"]["active"][0]]
else:
clouds = [cloud]
for cloud in clouds:
config = ConfigDict("cloudmesh.yaml")
cloud_config = config["cloudmesh.clouds"][cloud]
# Checking credentials
print("Checking cloud credentials...")
for prop in cloud_config["credentials"]:
if cloud_config["credentials"][prop].lower() in values_to_replace:
value = input(prop + "(" + cloud_config["credentials"][prop] + "): ")
cloud_config["credentials"][prop] = value
# Checking defaults
print("Checking cloud defaults...")
for prop in cloud_config["default"]:
if cloud_config["default"][prop].lower() in values_to_replace:
value = input(prop + "(" + cloud_config["default"][prop] + "): ")
cloud_config["default"][prop] = value
config.save()
export(cloud, "table")
return ""
elif arguments['user']:
username = arguments["USERNAME"] or getpass.getuser()
CloudRegister.set_username(username)
Console.ok("Setting profile user to {} in the yaml file.".format(username))
hosts = ssh_config()
hosts.generate(key="india", username=username, verbose=True)
return ""
elif arguments['ENTRY'].lower() in ['chameleon']:
config = ConfigDict("cloudmesh.yaml")
credentials = dotdict(config["cloudmesh.clouds.chameleon.credentials"])
default = credentials.OS_USERNAME
username = input("Please enter the username for {:} [{}]: ".format("chameleon",
default))
username = username or default
while True:
default = credentials.OS_PROJECT_NAME
project = input("Please enter the project id for {:} [{}]: ".format("chameleon",
default))
project = project or default
if project.isdigit():
project = "CH-{}".format(project)
break
else:
try:
prefix, number = project.split("-")
if not (prefix in ["CH"] and number.isdigit()):
Console.error("This is not a valid Chameleon.org cloud project", traceflag=False)
else:
break
except:
Console.error("This is not a valid Chameleon.org cloud project", traceflag=False)
password = getpass.getpass("Please enter the password for {:}: ".format("chameleon", credentials.OS_PASSWORD))
credentials.OS_TENENT_ID = credentials.OS_PROJECT_NAME
credentials.OS_TENENT_NAME = credentials.OS_PROJECT_NAME
credentials.OS_USERNAME = username
credentials.OS_PASSWORD = password
config.save()
return ""
elif arguments['ENTRY'] is not None:
name = arguments['ENTRY']
Register.entry(name)
return ""
# if all fails do a simple list
filename = _get_config_yaml_file(arguments)
CloudRegister.list(filename)
pass
def boot_vm(self,
name,
group=None,
image=None,
flavor=None,
cloud=None,
cert_thumbprint=None,
pub_key_path=None,
cert_path=None,
pfx_path=None,
secgroup=None,
meta=None,
nics=None,
**kwargs):
"""
Boots up a new VM Instance.
Steps involved: creating a hosted(Cloud) Service, adding the PFX certificate file,
get default storage name, creating a configuration set, adding an endpoint(SSH by default),
and finally creating a VM deployment
:param name: Hosted Service Name and VM instance name
:param group:
:param image:
:param flavor:
:param cloud:
:param cert_thumbprint:
:param pub_key_path:
:param cert_path:
:param pfx_path:
:param secgroup:
:param meta:
:param nics:
:param kwargs:
:return:
"""
location = ConfigDict(filename="cloudmesh.yaml")["cloudmesh"]["clouds"]["azure"]["default"]["location"] or 'Central US'
try:
self.provider.create_hosted_service(service_name=name,
label=name,
location=location)
except:
Console.error("Failed to create hosted service in Azure: {0}".format(traceback.format_exc()))
try:
Console.info("service name: " + name)
Console.info("location name: " + location)
Console.info("cert_thumbprint: " + cert_thumbprint)
Console.info("pub_key_path: " + pub_key_path)
Console.info("cert_path: " + cert_path)
Console.info("pfx_path:" + pfx_path)
Console.info("Image:" + image)
Console.info("Flavor:" + flavor)
#Console.info("Certificate adding")
# Disabled - not required to start Virtual Machine
#self.add_certificate(name, pfx_path)
#Console.info("Certificate added")
except Exception as e:
Console.warning("Console.info error: {0}".format(traceback.format_exc()))
storage_name = self._get_storage_name()
if storage_name is None:
self._create_storage_account()
storage_name = self._get_storage_name()
media_link = 'https://{0}.blob.core.windows.net/vhds/{1}.vhd'.format(
storage_name, name)
os_hd = OSVirtualHardDisk(image, media_link)
username = ConfigDict(filename="cloudmesh.yaml")["cloudmesh"]["clouds"]["azure"]["default"]["username"]
password = ConfigDict(filename="cloudmesh.yaml")["cloudmesh"]["clouds"]["azure"]["default"]["password"]
# Auto-generated Password in case of TBD
if username.lower() in ["tbd"]:
username = "******";
if password.lower() in ["tbd"]:
password = generate_password(16)
Console.info("Username: "******"password: "******"blob storage location: {0} ".format(media_link))
try:
vm_create_result = self.provider.create_virtual_machine_deployment(service_name=name,
deployment_name=name,
deployment_slot='production',
label=name,
role_name=name,
system_config=linux_config,
os_virtual_hard_disk=os_hd,
network_config=network,
role_size=flavor)
# pprint(vm_create_result)
self.provider.wait_for_operation_status(vm_create_result.request_id, timeout=30)
Console.info("{0} created successfully".format(name))
except:
Console.error("Failed to start Azure Virtual Machine: {0}".format(traceback.format_exc()))
return name
def from_environ(cls, provider):
"""
Reads env OS_* variables and registers a new cloud in yaml, interactively.
:return:
"""
yaml_data = ConfigDict("cloudmesh.yaml")
env_config_data = cls.read_env_config()
if env_config_data["OS_AUTH_URL"] is None:
print("ERROR: Cloud credentials not set in environment")
return
cloudname_suggest = urlparse(env_config_data["OS_AUTH_URL"]).hostname
# Command line inputs
cloudname_to_use = input(
"Name of the cloud (Default: {:}): ".format(
cloudname_suggest)) or cloudname_suggest
cm_label = input(
"Label for the cloud (Default: {:}): ".format(cloudname_to_use)) or "{:}".format(cloudname_to_use)
cm_heading = input(
"Heading for the cloud (Default: {:} Cloud): ".format(cm_label)) or "{:} Cloud".format(cm_label)
cm_host = input("Cloud host name (Default: {:}): ".format(cloudname_suggest)) or "{:}" \
.format(cloudname_suggest)
if provider is None:
# TODO: Check if the suggestion can be determined dynamically
cm_type = input("Type of the cloud- openstack/azure/ec2 "
"(Default: openstack): ") or "openstack"
else:
cm_type = provider
while cm_type not in ["openstack", "azure", "ec2"]:
print("\nType of cloud '{:}' is invalid and should be one "
"of openstack/ azure/ ec2.\n"
.format(cm_type))
cm_type = input("Type of the cloud- openstack/azure/ec2 "
"(Default: openstack): ") or "openstack"
cm_type_version = input(
"Version of type {:} (Default: null): ".format(cm_type)) or None
# Populate the dict with the data fetched from env
yaml_data["cloudmesh"]["clouds"][cloudname_to_use] = \
{"cm_heading": cm_heading,
"cm_host": cm_host,
"cm_label": cm_label,
"cm_type": cm_type,
"cm_type_version": cm_type_version,
"credentials": env_config_data
}
# Get defaults from user
default_flavor = input("Default flavor for the cloud instances"
"(Default: null): ") or None
default_image = input("Default image for the cloud instances"
" (Default: null): ") or None
default_location = input(
"Default location for the cloud instances "
"(Default: null): ") or None
yaml_data["cloudmesh"]["clouds"][cloudname_to_use]["default"] = \
{"flavor": default_flavor,
"image": default_image,
"location": default_location
}
# Save data in yaml
yaml_data.save()
print("New cloud config exported to {:}".format(yaml_data.filename))
def add(cls, name=None, type="vm", id=None, category="kilo"):
"""
Add an instance to a new group
or add it to an existing one
:param name:
:param type:
:param id:
:param cloud:
:return:
"""
# user logged into cloudmesh
user = ConfigDict.getUser(category) or cls.cm.user
try:
# See if group already exists. If yes, add id to the group
query = {
'name': name,
'category': category
}
# Find an existing group with name
existing_group = cls.cm.find("group", output="object",
**query).first()
# Existing group
if existing_group is not None:
id_str = str(existing_group.value)
ids = id_str.split(',')
# check if id is already in group
if id in ids:
Console.error("ID [{}] is already part of Group [{}]"
.format(id, name))
else:
id_str += ',' + id # add the id to the group
existing_group.value = id_str
cls.cm.save()
Console.ok("Added ID [{}] to Group [{}]"
.format(id, name))
# Create a new group
else:
obj_d = cls.cm.db_obj_dict("group",
name=name,
value=id,
type=type,
category=category,
user=user)
cls.cm.add_obj(obj_d)
cls.cm.save()
"""
group_obj = model.GROUP(
name,
id,
type,
category=category,
user=user
)
cls.cm.add(group_obj)
cls.cm.save()
"""
Console.ok("Created a new group [{}] and added ID [{}] to it"
.format(name, id))
except Exception as ex:
Console.error(ex.message, ex)
return
def do_register(self, args, arguments):
"""
::
Usage:
register info
register new
register clean [--force]
register list ssh [--format=FORMAT]
register list [--yaml=FILENAME][--info][--format=FORMAT]
register cat [--yaml=FILENAME]
register edit [--yaml=FILENAME]
register export HOST [--password] [--format=FORMAT]
register source HOST
register merge FILEPATH
register form [--yaml=FILENAME]
register check [--yaml=FILENAME]
register test [--yaml=FILENAME]
register json HOST
register remote [CLOUD] [--force]
register env [--provider=PROVIDER]
register profile --username=[USERNAME]
register yaml ENTRY
register CLOUD [--force]
register CLOUD [--dir=DIR]
register ec2 CLOUD EC2ZIP
managing the registered clouds in the cloudmesh.yaml file.
It looks for it in the current directory, and than in
~/.cloudmesh. If the file with the cloudmesh.yaml name is
there it will use it. If neither location has one a new
file will be created in ~/.cloudmesh/cloudmesh.yaml. Some
defaults will be provided. However you will still need to
fill it out with valid entries.
Arguments:
HOST the host name
USER the user name
FILEPATH the path of the file
CLOUD the cloud name
PROVIDER the provider or type of cloud [Default: openstack]
USERNAME Username that would be registered in yaml. Defaults to OS username.
Options:
--provider=PROVIDER Provider to be used for cloud. Values are:
openstack, azure, ec2.
--version=VERSION Version of the openstack cloud.
--openrc=OPENRC The location of the openrc file
--password Prints the password
--force ignore interactive questions and execute
the action
Description:
register info
It looks out for the cloudmesh.yaml file in the current
directory, and then in ~/.cloudmesh
register list [--yaml=FILENAME] [--name] [--info]
lists the clouds specified in the cloudmesh.yaml file. If
info is specified it also prints the location of the yaml
file.
register list ssh
lists hosts from ~/.ssh/config
register cat [--yaml=FILENAME]
outputs the cloudmesh.yaml file
register edit [--yaml=FILENAME]
edits the cloudmesh.yaml file
register export HOST [--format=FORMAT]
prints the contents of an openrc.sh file based on the
information found in the cloudmesh.yaml file.
register remote CLOUD [--force]
reads the Openstack OPENRC file from a remote host that
is described in cloudmesh.yaml file. We assume that
the file has already a template for this host. If
not it can be created from other examples before
you run this command.
It uses the OS_OPENRC variable to locate the file and
copy it onto your computer.
register merge FILENAME
Replaces the TBD in cloudmesh.yaml with the contents
present in the named file
register form [--yaml=FILENAME]
interactively fills out the form wherever we find TBD.
register check [--yaml=FILENAME]
checks the yaml file for completness
register test [--yaml=FILENAME]
checks the yaml file and executes tests to check if
we can use the cloud. TODO: maybe this should be in
a test command
register json host
displays the host details in json format
register remote CLOUD
registers a remote cloud and copies the openrc file
specified in the credentials of the cloudmesh.yaml
register CLOUD --dir
Copies the entire directory from the cloud and puts it in
~/.cloudmesh/clouds/host
For kilo, The directory would be copied to
~/.cloudmesh/clouds/kilo
register env [--provider=PROVIDER] [HOSTNAME]
Reads env OS_* variables and registers a new cloud in yaml,
interactively. Default PROVIDER is openstack and HOSTNAME
is localhost.
register username [USERNAME]
Sets the username in yaml with the value provided.
"""
# from pprint import pprint
# pprint(arguments)
def _get_config_yaml_file(arguments):
filename = arguments["--yaml"] or "cloudmesh.yaml"
filename = Config.find_file(filename)
return filename
def exists(filename):
return os.path.isfile(filename)
def export(host, output):
config = ConfigDict("cloudmesh.yaml")
credentials = dict(
config["cloudmesh"]["clouds"][host]["credentials"])
if not arguments["--password"]:
credentials["OS_PASSWORD"] = "******"
if output is None:
for attribute, value in credentials.items():
print("export {}={}".format(attribute, value))
elif output == "table":
print(attribute_printer(credentials))
else:
print(dict_printer(credentials, output=output))
# TODO: bug csv does not work
return ""
if arguments["info"]:
filename = _get_config_yaml_file(arguments)
if os.path.isfile(filename):
Console.ok("File '{}' exists. ok.".format(filename))
Console.ok("The yaml file contains the following templates:")
d = CloudRegister.list(filename,
info=False,
output="table")
print(d)
else:
Console.error("File {} does not exist".format(filename))
return ""
elif arguments["new"]:
import shutil
import cloudmesh_client.etc
print(cloudmesh_client.etc.__file__)
filename = os.path.join(
os.path.dirname(cloudmesh_client.etc.__file__),
"cloudmesh.yaml")
Console.ok(filename)
yamlfile = path_expand("~/.cloudmesh/cloudmesh.yaml")
shutil.copyfile(filename, yamlfile)
print("copy ")
print("From: ", filename)
print("To: ", yamlfile)
# filename = _get_config_yaml_file(arguments)
# if _exists(filename):
# Console.ok("File '{}' exists. ok.".format(filename))
# else:
# Console.error("File {} does not exist".format(filename))
return ""
elif arguments["clean"]:
filename = _get_config_yaml_file(arguments)
force = arguments["--force"] or False
if filename is not None:
print(filename, force)
if exists(filename):
print("Delete cloudmesh.yaml file:", filename)
if not force:
force = yn_choice("Would you like to delete the "
"cloudmesh.yaml file")
print(force)
if force:
os.remove(filename)
Console.ok("Deleted the file " + filename + ". ok.")
else:
Console.ok("Please use Y to delete the file.")
pass
else:
Console.error("File {} does not exist".format(filename))
else:
Console.error("No cloudmesh.yaml file found.")
return ""
elif arguments["cat"]:
filename = _get_config_yaml_file(arguments)
if exists(filename):
with open(filename, 'r') as f:
lines = f.read().split("\n")
print('\n'.join(lines))
else:
Console.error("File {} does not exist".format(filename))
return ""
elif arguments["edit"]:
filename = _get_config_yaml_file(arguments)
if exists(filename):
try:
data = {"editor": os.environ["EDITOR"],
"filename": filename}
Console.ok("editing file " + filename)
os.system("{editor} {filename}".format(**data))
except:
Console.error("No EDITOR variable set in shell.")
else:
Console.error("File {} does not exist".format(filename))
return ""
elif arguments['list'] and arguments['ssh']:
output = arguments['--format'] or 'table'
hosts = CloudRegister.list_ssh()
print(print_list(hosts, output=output))
return ""
elif arguments['list']:
filename = _get_config_yaml_file(arguments)
info = arguments["--info"] or False
output = arguments["--format"] or "table"
if not filename:
Console.error("File {} doesn't exist".format(filename))
else:
d = CloudRegister.list(filename,
info=info,
output=output)
print(d)
return ""
elif arguments['check']:
filename = _get_config_yaml_file(arguments)
if not filename:
Console.error("File {} doesn't exist".format(
arguments["--yaml"] or 'cloudmesh.yaml'))
else:
CloudRegister.check_yaml_for_completeness(filename)
return ""
elif arguments['merge']:
filename = arguments['FILENAME']
CloudRegister.from_file(filename)
return
elif arguments['test']:
filename = _get_config_yaml_file(arguments)
CloudRegister.test(filename)
return ""
elif arguments['form']:
filename = _get_config_yaml_file(arguments)
if not filename:
Console.error("File {} doesn't exist".format(
arguments["--yaml"] or 'cloudmesh.yaml'))
else:
CloudRegister.fill_out_form(filename)
return ""
elif arguments['source']:
host = arguments['HOST']
config = ConfigDict("cloudmesh.yaml")
credentials = dict(
config["cloudmesh"]["clouds"][host]["credentials"])
# unset
variables = list(os.environ)
for attribute in variables:
if attribute.startswith("OS_"):
print("x ", attribute)
del os.environ[attribute]
# set
for attribute, value in credentials.items():
os.putenv(attribute, value)
print("+ ", attribute)
export(host, "table")
return ""
elif arguments['export']:
output = arguments['--format']
host = arguments['HOST']
variables = list(os.environ)
for attribute in variables:
if attribute.startswith("OS_"):
print("unset ", attribute)
del os.environ[attribute]
export(host, output)
elif arguments['json']:
host = arguments['HOST']
result = CloudRegister.get(host)
if result:
print(json.dumps(result, indent=4))
else:
print("Cloud {:} is not described in cloudmesh.yaml".format(
host))
return ""
elif arguments['remote']:
force = arguments['--force']
cloud = arguments['CLOUD']
if cloud is None:
clouds = ["kilo"]
else:
clouds = [cloud]
for cloud in clouds:
CloudRegister.remote(cloud, force)
export(cloud, "table")
config = ConfigDict("cloudmesh.yaml")
if config["cloudmesh.profile.username"] == "TBD":
name = config["cloudmesh.clouds.kilo.credentials.OS_USERNAME"]
config["cloudmesh"]["profile"]["username"] = name
config.save()
else:
print("KKK")
return ""
elif arguments['ec2']:
cloud = arguments['CLOUD']
zipfile = arguments['EC2ZIP']
if cloud is None:
clouds = ["kilo"]
else:
clouds = [cloud]
for cloud in clouds:
CloudRegister.ec2(cloud, zipfile)
export(cloud, "table")
elif arguments['env']:
try:
CloudRegister.from_environ(arguments['--provider'])
except Exception as e:
Error.traceback(e)
return ""
elif arguments['CLOUD']:
if arguments['--dir']:
cloud = arguments['CLOUD']
directory = arguments['--dir']
Console.ok(directory)
CloudRegister.directory(cloud, directory)
else:
cloud = arguments['CLOUD']
if cloud is None:
clouds = ["kilo"]
else:
clouds = [cloud]
for cloud in clouds:
CloudRegister.remote(cloud, True)
export(cloud, "table")
return ""
elif arguments['profile']:
username = arguments["--username"] or getpass.getuser()
CloudRegister.set_username(username)
Console.ok("Username {} set successfully in the yaml settings.".format(username))
return ""
elif arguments['yaml']:
name = arguments['ENTRY']
Register.entry(name)
return ""
# if all fails do a simple list
filename = _get_config_yaml_file(arguments)
CloudRegister.list(filename)
pass
def __init__(self, context):
cmd.Cmd.__init__(self)
self.variables = {}
self.command_topics = {}
self.register_topics()
self.context = context
# TODO get loglevel from DB or yaml file, if not defined set to ERROR
self.loglevel = "DEBUG"
self._hist = []
if self.context.debug:
print("init CloudmeshConsole")
self.prompt = 'cm> '
self.doc_header = "Documented commands (type help <command>):"
self.banner = textwrap.dedent("""
+=======================================================+
. ____ _ _ _ .
. / ___| | ___ _ _ __| |_ __ ___ ___ ___| |__ .
. | | | |/ _ \| | | |/ _` | '_ ` _ \ / _ \/ __| '_ \ .
. | |___| | (_) | |_| | (_| | | | | | | __/\__ \ | | | .
. \____|_|\___/ \__,_|\__,_|_| |_| |_|\___||___/_| |_| .
+=======================================================+
Cloudmesh Shell
""")
# KeyCommands.__init__(self, context)
#
# set default cloud and default group if they do not exist
# use the first cloud in cloudmesh.yaml as default
#
filename = path_expand("~/.cloudmesh/cloudmesh.yaml")
create_cloudmesh_yaml(filename)
# Initialize Logging
# LogUtil.initialize_logging()
# sys,exit(1)
value = Default.get('cloud', category='general')
if value is None:
clouds = ConfigDict(filename=filename)["cloudmesh"]["clouds"]
cloud = clouds.keys()[0]
Default.set('cloud', cloud, category='general')
value = Default.get('default', category='general')
if value is None:
Default.set('default', 'default', category='general')
cluster = 'kilo' # hardcode a value if not defined
value = Default.get('cluster', category='general')
if value is None:
try:
hosts = ssh_config().names()
if hosts is not None:
cluster = hosts[0]
except:
pass # use the hardcoded cluster
else:
cluster = value
Default.set('cluster', cluster, category='general')
group = Default.get_group()
if group is None:
Default.set_group("default")
Default.load("cloudmesh.yaml")
on = Default.timer()
group = Default.get_group()
if group is None:
Default.set_group("default")
r = Default.get_refresh()
if r is None:
Default.set_refresh("on")
"""
try:
sshm = SSHKeyManager()
m = sshm.get_from_yaml(
load_order="~/.cloudmesh/cloudmesh.yaml")
d = dict(m.__keys__)
sshdb = SSHKeyDBManager()
for keyname in m.__keys__:
filename = m[keyname]["path"]
try:
sshdb.add(filename,
keyname,
source="yaml",
uri="file://" + filename)
except Exception as e:
pass
except Exception as e:
Console.error("Problem adding keys from yaml file")
"""
for c in CloudmeshConsole.__bases__[1:]:
# noinspection PyArgumentList
c.__init__(self, context)
def remote(cls, host, force=False):
"""
TODO: there is a bug in the instalation of kilo the openrc file on
the remote machine is not called openrc.sh but contains username and
project number.
:param host: the remote host
:param force:
:return:
"""
config = ConfigDict("cloudmesh.yaml")
host_spec = config["cloudmesh.clouds." + host]
host_credentials = host_spec["credentials"]
if 'cm_openrc' in host_spec:
Console.ok("looking for openrc")
else:
Console.error("no cm_openrc specified in the host")
return
hostname = config["cloudmesh.clouds." + host + ".cm_host"]
Console.ok("fetching information from {:} ...".format(host))
openrc = host_spec["cm_openrc"]
directory = os.path.dirname(openrc)
base = os.path.basename(openrc)
_from_dir = "{:}:{:}".format(hostname, directory + "/*").replace("~/", "")
# _to_dir = os.path.dirname(Config.path_expand(directory))
# FIX: Issues with path expanding on Windows
_to_dir = os.path.realpath(
os.path.expanduser(directory)
)
'''
In Windows, SCP fails with path such as C:\\Users\\...,
and passes with '~/.cloudmesh/...'
But on Linux machines, it fails with ~/.cloudmesh/...
and passes with /home/user/...
Hence, adding OS check below for SCP copy directory
'''
os_type = platform.system().lower()
if 'windows' not in os_type:
directory = _to_dir
# FIX: fix for scp not working on Windows, because scp does not
# understand
# paths in format: "C:/Users/<>", rather expects "~/.cloudmesh/<>"
# openrc_file = Config.path_expand(openrc)
openrc_file = os.path.realpath(
os.path.expanduser(openrc)
)
print("From: ", _from_dir)
print("To: ", _to_dir)
print("Openrc:", openrc_file)
cls.make_dir(_to_dir)
r = ""
Console.ok("Reading rc file from {}".format(host))
try:
r = Shell.scp('-r', _from_dir, directory)
except Exception as e:
print(e)
return
#
# TODO: the permission are not yet right
#
os.chmod(_to_dir, 0o700)
for root, dirs, _ in os.walk(_to_dir):
for d in dirs:
os.chmod(os.path.join(root, d), 0o700)
#
# END PERMISSION
#
with open(openrc_file, 'r') as f:
lines = f.read().split("\n")
config = ConfigDict("cloudmesh.yaml")
for line in lines:
if line.strip().startswith("export"):
line = line.replace("export ", "")
key, value = line.split("=", 1)
config["cloudmesh"]["clouds"][host]["credentials"][key] = value
host_spec = config["cloudmesh"]["clouds"][host]
credentials = host_spec["credentials"]
if "cm_openrc" in host_spec:
openrc = host_spec["cm_openrc"]
for attribute in credentials:
if attribute in openrc:
openrc.replace(attribute, credentials[attribute])
config.save()
config = ConfigDict("cloudmesh.yaml")
return config["cloudmesh"]["clouds"][host]["credentials"]
def do_comet(self, args, arguments):
"""
::
Usage:
comet init
comet ll [CLUSTERID] [--format=FORMAT]
comet cluster [CLUSTERID]
[--format=FORMAT]
comet computeset [COMPUTESETID]
[--allocation=ALLOCATION]
[--cluster=CLUSTERID]
[--state=COMPUTESESTATE]
comet start CLUSTERID [--count=NUMNODES] [COMPUTENODEIDS]
[--allocation=ALLOCATION]
[--walltime=WALLTIME]
comet terminate COMPUTESETID
comet power (on|off|reboot|reset|shutdown) CLUSTERID [NODESPARAM]
comet console CLUSTERID [COMPUTENODEID]
comet iso list
comet iso upload [--isoname=ISONAME] PATHISOFILE
comet iso attach ISONAME CLUSTERID [COMPUTENODEIDS]
comet iso detach CLUSTERID [COMPUTENODEIDS]
comet node rename CLUSTERID OLDNAME NEWNAME
Options:
--format=FORMAT Format is either table, json, yaml,
csv, rest
[default: table]
--count=NUMNODES Number of nodes to be powered on.
When this option is used, the comet system
will find a NUMNODES number of arbitrary nodes
that are available to boot as a computeset
--allocation=ALLOCATION Allocation to charge when power on
node(s)
--walltime=WALLTIME Walltime requested for the node(s).
Walltime could be an integer value followed
by a unit (m, h, d, w, for minute, hour, day,
and week, respectively). E.g., 3h, 2d
--isoname=ISONAME Name of the iso image after being stored remotely.
If not specified, use the original filename
--state=COMPUTESESTATE List only computeset with the specified state.
The state could be submitted, running, completed
Arguments:
CLUSTERID The assigned name of a cluster, e.g. vc1
COMPUTESETID An integer identifier assigned to a computeset
COMPUTENODEID A compute node name, e.g., vm-vc1-0
If not provided, the requested action will be taken
on the frontend node of the specified cluster
COMPUTENODEIDS A set of compute node names in hostlist format,
e.g., vm-vc1-[0-3]
One single node is also acceptable: vm-vc1-0
If not provided, the requested action will be taken
on the frontend node of the specified cluster
NODESPARAM Specifying the node/nodes/computeset to act on.
In case of integer, will be intepreted as a computesetid;
in case of a hostlist format, e.g., vm-vc1-[0-3], a group
of nodes; or a single host is also acceptable,
e.g., vm-vc1-0
ISONAME Name of an iso image at remote server
PATHISOFILE The full path to the iso image file to be uploaded
"""
# back up of all the proposed commands/options
"""
comet status
comet tunnel start
comet tunnel stop
comet tunnel status
comet logon
comet logoff
comet ll [CLUSTERID] [--format=FORMAT]
comet docs
comet info [--user=USER]
[--project=PROJECT]
[--format=FORMAT]
comet cluster [CLUSTERID][--name=NAMES]
[--user=USER]
[--project=PROJECT]
[--hosts=HOSTS]
[--start=TIME_START]
[--end=TIME_END]
[--hosts=HOSTS]
[--format=FORMAT]
comet computeset [COMPUTESETID]
comet start ID
comet stop ID
comet power on CLUSTERID [NODESPARAM]
[--allocation=ALLOCATION]
[--walltime=WALLTIME]
comet power (off|reboot|reset|shutdown) CLUSTERID [NODESPARAM]
comet console CLUSTERID [COMPUTENODEID]
comet delete [all]
[--user=USER]
[--project=PROJECT]
[--name=NAMES]
[--hosts=HOSTS]
[--start=TIME_START]
[--end=TIME_END]
[--host=HOST]
comet delete --file=FILE
comet update [--name=NAMES]
[--hosts=HOSTS]
[--start=TIME_START]
[--end=TIME_END]
comet add [--user=USER]
[--project=PROJECT]
[--host=HOST]
[--description=DESCRIPTION]
[--start=TIME_START]
[--end=TIME_END]
NAME
comet add --file=FILENAME
Options:
--user=USER user name
--name=NAMES Names of the vcluster
--start=TIME_START Start time of the vcluster, in
YYYY/MM/DD HH:MM:SS format.
[default: 1901-01-01]
--end=TIME_END End time of the vcluster, in YYYY/MM/DD
HH:MM:SS format. In addition a duratio
can be specified if the + sign is the
first sig The duration will than be
added to the start time.
[default: 2100-12-31]
--project=PROJECT project id
--host=HOST host name
--description=DESCRIPTION description summary of the vcluster
--file=FILE Adding multiple vclusters from one file
--format=FORMAT Format is either table, json, yaml,
csv, rest
[default: table]
--allocation=ALLOCATION Allocation to charge when power on
node(s)
--walltime=WALLTIME Walltime requested for the node(s)
Arguments:
FILENAME the file to open in the cwd if . is
specified. If file in in cwd
you must specify it with ./FILENAME
Opens the given URL in a browser window.
"""
"""
if not arguments["tunnel"] and Comet.tunnelled and not Comet.is_tunnel():
Console.error("Please establish a tunnel first with:")
print
print (" comet tunnel start")
print
return ""
try:
if not arguments["tunnel"]:
logon = Comet.logon()
if logon is False:
Console.error("Could not logon")
return ""
except:
Console.error("Could not logon")
# pprint (arguments)
output_format = arguments["--format"] or "table"
if arguments["status"]:
Comet.state()
elif arguments["tunnel"] and arguments["start"]:
Comet.tunnel(True)
elif arguments["tunnel"] and arguments["stop"]:
Comet.tunnel(False)
elif arguments["tunnel"] and arguments["status"]:
Comet.state()
elif arguments["logon"]:
if self.context.comet_token is None:
if Comet.logon():
Console.ok("logging on")
self.context.comet_token = Comet.token
else:
Console.error("could not logon")
else:
Console.error("already logged on")
elif arguments["logoff"]:
if self.context.comet_token is None:
Console.error("not logged in")
else:
if Comet.logoff():
Console.ok("Logging off")
self.context.comet_token = None
else:
Console.error(
"some issue while logging off. Maybe comet not reachable")
elif arguments["docs"]:
Comet.docs()
elif arguments["info"]:
Console.error("not yet implemented")
elif arguments["add"]:
print ("add the cluster")
elif arguments["start"]:
cluster_id = arguments["ID"]
print("start", cluster_id)
Cluster.start(cluster_id)
elif arguments["stop"]:
cluster_id = arguments["ID"]
print("stop", cluster_id)
Cluster.stop(cluster_id)
elif arguments["ll"]:
"""
if arguments["init"]:
print ("Initializing the comet configuration file...")
config = ConfigDict("cloudmesh.yaml")
# for unit testing only.
cometConf = config["cloudmesh.comet"]
endpoints = []
# print (cometConf.keys())
if "endpoints" in cometConf.keys():
endpoints = cometConf["endpoints"].keys()
if len(endpoints) < 1:
Console.error("No service endpoints available."\
" Please check the config template")
return ""
if "username" in cometConf.keys():
default_username = cometConf['username']
# print (default_username)
if 'TBD' == default_username:
set_default_user = \
input("Set a default username (RETURN to skip): ")
if set_default_user:
config.data["cloudmesh"]["comet"]["username"] = \
set_default_user
config.save()
Console.ok("Comet default username set!")
if "active" in cometConf.keys():
active_endpoint = cometConf['active']
set_active_endpoint = \
input("Set the active service endpoint to use. "\
"The availalbe endpoints are - %s [%s]: "\
% ("/".join(endpoints),
active_endpoint)
)
if set_active_endpoint:
if set_active_endpoint in endpoints:
config.data["cloudmesh"]["comet"]["active"] = \
set_active_endpoint
config.save()
Console.ok("Comet active service endpoint set!")
else:
Console.error("The provided endpoint does not match any "\
"available service endpoints. Try %s" \
% "/".join(endpoints) )
if cometConf['active'] in endpoints:
endpoint_url = cometConf["endpoints"]\
[cometConf['active']]["nucleus_base_url"]
api_version = cometConf["endpoints"]\
[cometConf['active']]["api_version"]
set_endpoint_url = \
input("Set the base url for the nucleus %s service [%s]: "\
% (cometConf['active'],
endpoint_url)
)
if set_endpoint_url:
if set_endpoint_url != endpoint_url:
config.data["cloudmesh"]["comet"]["endpoints"]\
[cometConf['active']]["nucleus_base_url"]\
= set_endpoint_url
config.save()
Console.ok("Service base url set!")
set_api_version = \
input("Set the api version for the nucleus %s service [%s]: "\
% (cometConf['active'],
api_version)
)
if set_api_version:
if set_api_version != api_version:
config.data["cloudmesh"]["comet"]["endpoints"]\
[cometConf['active']]["api_version"]\
= set_api_version
config.save()
Console.ok("Service api version set!")
print ("Authenticating to the nucleus %s "\
"service and obtaining the apikey..." \
% cometConf['active'])
Comet.get_apikey(cometConf['active'])
return ''
#Comet.get_apikey()
try:
logon = Comet.logon()
if logon is False:
Console.error("Could not logon. Please try first:\ncm comet init")
return ""
except:
Console.error("Could not logon")
output_format = arguments["--format"] or "table"
if arguments["ll"]:
cluster_id = arguments["CLUSTERID"] or None
print(Cluster.simple_list(cluster_id, format=output_format))
elif arguments["cluster"]:
cluster_id = arguments["CLUSTERID"]
print(Cluster.list(cluster_id, format=output_format))
elif arguments["computeset"]:
computeset_id = arguments["COMPUTESETID"] or None
cluster = arguments["--cluster"] or None
state = arguments["--state"] or None
allocation = arguments["--allocation"] or None
cluster = arguments["--cluster"] or None
print (Cluster.computeset(computeset_id, cluster, state, allocation))
elif arguments["start"]:
clusterid = arguments["CLUSTERID"]
numnodes = arguments["--count"] or None
computenodeids = arguments["COMPUTENODEIDS"] or None
# check allocation information for the cluster
cluster = Cluster.list(clusterid, format='rest')
try:
allocations = cluster[0]['allocations']
except:
# print (cluster)
Console.error("No allocation available for the specified cluster." \
"Please check with the comet help team")
return ""
# checking whether the computesetids is in valid hostlist format
if computenodeids:
try:
hosts_param = hostlist.expand_hostlist(computenodeids)
except hostlist.BadHostlist:
Console.error("Invalid hosts list specified!",
traceflag=False)
return ""
elif numnodes:
try:
param = int(numnodes)
except ValueError:
Console.error("Invalid count value specified!", traceflag=False)
return ""
if param <= 0:
Console.error("count value has to be greather than zero")
return ""
numnodes = param
else:
Console.error("You have to specify either the count of nodes, " \
"or the names of nodes in hostlist format")
return ""
walltime = arguments["--walltime"] or None
allocation = arguments["--allocation"] or None
# validating walltime and allocation parameters
walltime = Cluster.convert_to_mins(walltime)
if not walltime:
print ("No valid walltime specified. "\
"Using system default (2 days)")
if not allocation:
if len(allocations) == 1:
allocation = allocations[0]
else:
allocation = Cluster.display_get_allocation(allocations)
# issuing call to start a computeset with specified parameters
print (Cluster.computeset_start(clusterid,
computenodeids,
numnodes,
allocation,
walltime)
)
elif arguments["terminate"]:
computesetid = arguments["COMPUTESETID"]
print (Cluster.computeset_terminate(computesetid))
elif arguments["power"]:
clusterid = arguments["CLUSTERID"] or None
fuzzyparam = arguments["NODESPARAM"] or None
# parsing nodesparam for proper action
if fuzzyparam:
try:
param = int(fuzzyparam)
subject = 'COMPUTESET'
except ValueError:
param = fuzzyparam
try:
hosts_param = hostlist.expand_hostlist(fuzzyparam)
subject = 'HOSTS'
except hostlist.BadHostlist:
Console.error("Invalid hosts list specified!",
traceflag=False)
return ""
else:
subject = 'FE'
param = None
if arguments["on"]:
action = "on"
elif arguments["off"]:
action = "off"
elif arguments["reboot"]:
action = "reboot"
elif arguments["reset"]:
action = "reset"
elif arguments["shutdown"]:
action = "shutdown"
else:
action = None
print (Cluster.power(clusterid,
subject,
param,
action)
)
elif arguments["console"]:
clusterid = arguments["CLUSTERID"]
nodeid = None
if 'COMPUTENODEID' in arguments:
nodeid = arguments["COMPUTENODEID"]
Comet.console(clusterid, nodeid)
elif arguments["iso"]:
if arguments["list"]:
isos = (Comet.list_iso())
idx = 0
for iso in isos:
if iso.startswith("public/"):
iso = iso.split("/")[1]
idx += 1
print ("{}: {}".format(idx, iso))
if arguments["upload"]:
isofile = arguments["PATHISOFILE"]
isofile = os.path.abspath(isofile)
if os.path.isfile(isofile):
if arguments["--isoname"]:
filename = arguments["--isoname"]
else:
filename = os.path.basename(isofile)
else:
print ("File does not exist - {}"\
.format(arguments["PATHISOFILE"]))
return ""
print (Comet.upload_iso(filename, isofile))
elif arguments["attach"]:
isoname = arguments["ISONAME"]
clusterid = arguments["CLUSTERID"]
computenodeids = arguments["COMPUTENODEIDS"] or None
print (Cluster.attach_iso(isoname, clusterid, computenodeids))
elif arguments["detach"]:
clusterid = arguments["CLUSTERID"]
computenodeids = arguments["COMPUTENODEIDS"] or None
print (Cluster.detach_iso(clusterid, computenodeids))
elif arguments["node"]:
if arguments["rename"]:
clusterid = arguments["CLUSTERID"]
oldname = arguments["OLDNAME"]
newname = arguments["NEWNAME"]
if newname is None or newname == '':
print ("New node name cannot be empty")
else:
print (Cluster.rename_node(clusterid, oldname, newname))
'''
# bulk rename
if arguments["rename"]:
oldnames = Parameter.expand(arguments["OLDNAME"])
newnames = Parameter.expand(arguments["NEWNAME"])
# check if new names ar not already taken
# to be implemented
if len(oldnames) == len(newnames):
for i in range(0,len(oldnames)):
oldname = oldnames[i]
newname = newnames[i]
if newname is None or newname == '':
print ("New node name cannot be empty")
else:
print (Cluster.rename_node(clusterid, oldname, newname))
'''
return ""
