def get_from_dir(cls, directory=None, store=True):
directory = directory or Config.path_expand("~/.ssh")
files = [file for file in os.listdir(expanduser(Config.path_expand(directory)))
if file.lower().endswith(".pub")]
d = []
for file in files:
location = Config.path_expand("{:}/{:}".format(directory, file))
sshkey = SSHkey(location).get()
i = sshkey["comment"]
if i is not None:
i = i.replace("@", "_")
i = i.replace("-", "_")
i = i.replace(" ", "_")
i = i.replace(".", "_")
else:
# use base name
i = file.replace(".pub", "")
sshkey["kind"] = "key"
sshkey["source"] = 'file'
if store:
cls._add_from_sshkey(
dict(sshkey),
keyname=sshkey["name"],
source=sshkey["source"],
uri=sshkey["uri"])
else:
d.append(dict(sshkey))
if not store:
return d
def get_from_yaml(cls, filename=None, load_order=None, store=True):
"""
:param filename: name of the yaml file
:return: a SSHKeyManager (dict of keys)
"""
config = None
if filename is None:
# default = Config.path_expand(os.path.join("~", ".cloudmesh", "cloudmesh.yaml"))
# config = ConfigDict("cloudmesh.yaml")
filename = "cloudmesh.yaml"
config = ConfigDict(filename)
elif load_order:
config = ConfigDict(filename, load_order)
else:
Console.error("Wrong arguments")
return
config_keys = config["cloudmesh"]["keys"]
default = config_keys["default"]
keylist = config_keys["keylist"]
uri = Config.path_expand(os.path.join("~", ".cloudmesh", filename))
d = []
for key in list(keylist.keys()):
keyname = key
value = keylist[key]
if os.path.isfile(Config.path_expand(value)):
path = Config.path_expand(value)
if store:
Key.add_from_path(path, keyname)
else:
d.append(Key.add_from_path(path, keyname, store=False))
else:
keytype, string, comment = SSHkey._parse(value)
thekey = {
'uri': 'yaml://{}'.format(uri),
'string': value,
'fingerprint': SSHkey._fingerprint(value),
'name': keyname,
'comment': comment,
'source': 'git',
'kind': 'key'
}
thekey["type"], thekey["key"], thekey[
"comment"] = SSHkey._parse(value)
if thekey["comment"] is None:
thekey["comment"] = keyname
if store:
try:
cls.cm.add(thekey)
except:
Console.error("Key already in db", traceflag=False)
else:
d.append(thekey)
if not store:
return d
"""
def get_from_dir(cls, directory=None, store=True):
directory = directory or Config.path_expand("~/.ssh")
files = [
file
for file in os.listdir(expanduser(Config.path_expand(directory)))
if file.lower().endswith(".pub")
]
d = []
for file in files:
location = Config.path_expand("{:}/{:}".format(directory, file))
sshkey = SSHkey(location).get()
i = sshkey["comment"]
if i is not None:
i = i.replace("@", "_")
i = i.replace("-", "_")
i = i.replace(" ", "_")
i = i.replace(".", "_")
else:
# use base name
i = file.replace(".pub", "")
sshkey["kind"] = "key"
sshkey["source"] = 'file'
if store:
cls._add_from_sshkey(dict(sshkey),
keyname=sshkey["name"],
source=sshkey["source"],
uri=sshkey["uri"])
else:
d.append(dict(sshkey))
if not store:
return d
def get_from_dir(self, directory=None):
directory = directory or Config.path_expand("~/.ssh")
files = [file for file in os.listdir(expanduser(Config.path_expand(directory)))
if file.lower().endswith(".pub")]
for file in files:
location = Config.path_expand("{:}/{:}".format(directory, file))
sshkey = SSHkey(location)
i = sshkey.comment
self.__keys__[i] = sshkey.__key__
def add_from_path(cls,
path,
keyname=None,
user=None,
source=None,
uri=None,
store=True):
"""
Adds the key to the database based on the path
:param keyname: name of the key or path to the key
:return:
"""
user = user or cls.cm.user
sshkey = SSHkey(Config.path_expand(path))
if store:
cls._add_from_sshkey(sshkey.__key__,
keyname,
user,
source=source,
uri=uri)
else:
return sshkey.__key__
def get_hostname(cls, host):
"""
Method to return hostname
for a host in ssh config
:param host:
:return:
"""
filename = Config.path_expand("~/.ssh/config")
with open(filename, 'r') as f:
lines = f.read().split("\n")
found = False
for line in lines:
# search for host
if "Host " in line:
_host = line.strip().replace("Host ", "", 1).replace(" ", "")
# if host found in ssh config
if _host == host:
found = True
pass
# search for hostname
if "Hostname " in line and found is True:
# return corresponding hostname
hostname = line.strip().replace("Hostname ", "",
1).replace(" ", "")
return hostname
def read(self, file_path, keyname=None):
self.__key__ = {}
if file_path is not None:
file_path = Config.path_expand(file_path)
uri = 'file://{}'.format(file_path)
self.__key__ = {
'uri': uri,
'string': open(file_path, "r").read().rstrip()
}
(self.__key__['type'],
self.__key__['key'],
self.__key__['comment']) = self._parse(self.__key__['string'])
self.__key__['fingerprint'] = self._fingerprint(self.__key__['string'])
# Workaround for multiple file entries in cloudmesh.yaml getting same name derived from file name (like id_rsa).
# This caused the dict to have just 1 entry as the name is the key.
# Change tracked in git issue #8
if keyname is None:
name = basename(file_path).replace(".pub", "").replace("id_", "")
else:
name = keyname
self.__key__['name'] = name
self.__key__['comment'] = self.__key__['comment']
self.__key__['source'] = 'ssh'
return self.__key__
def get_hostuser(cls, host):
"""
Method to return user login
for a host in ssh config
:param host:
:return:
"""
filename = Config.path_expand("~/.ssh/config")
with open(filename, 'r') as f:
lines = f.read().split("\n")
found = False
for line in lines:
# search for host
if "Host " in line:
_host = line.strip().replace("Host ", "", 1).replace(" ", "")
# if host found in ssh config
if _host == host:
found = True
pass
# search for user
if "User " in line and found is True:
# return corresponding user
username = line.strip().replace("User ", "", 1).replace(" ", "")
return username
def add_from_path(cls,
path,
keyname=None,
user=None,
source=None,
uri=None,
store=True):
"""
Adds the key to the database based on the path
:param keyname: name of the key or path to the key
:return:
"""
user = user or cls.cm.user
sshkey = SSHkey(Config.path_expand(path))
if store:
cls._add_from_sshkey(sshkey.__key__,
keyname,
user,
source=source,
uri=uri)
else:
return sshkey.__key__
def read(self, file_path, keyname=None):
self.__key__ = {}
if file_path is not None:
orig_path = file_path
file_path = Config.path_expand(file_path)
uri = 'file://{}'.format(file_path)
self.__key__ = {
'uri': uri,
'path': orig_path,
'string': open(file_path, "r").read().rstrip()
}
(self.__key__['type'], self.__key__['key'],
self.__key__['comment']) = self._parse(self.__key__['string'])
self.__key__['fingerprint'] = self._fingerprint(
self.__key__['string'])
# Workaround for multiple file entries in cloudmesh.yaml getting same name derived from file name (like id_rsa).
# This caused the dict to have just 1 entry as the name is the key.
# Change tracked in git issue #8
if keyname is None:
name = basename(file_path).replace(".pub",
"").replace("id_", "")
else:
name = keyname
self.__key__['name'] = name
self.__key__['comment'] = self.__key__['comment']
self.__key__['source'] = 'ssh'
return self.__key__
def remote(cls, host, force=False):
"""
TODO: there is a bug in the instalation of kilo the openrc file on
the remote machine is not called openrc.sh but contains username and
project number.
:param host: the remote host
:param force:
:return:
"""
config = ConfigDict("cloudmesh.yaml")
host_spec = config["cloudmesh.clouds." + host]
host_credentials = host_spec["credentials"]
if 'cm_openrc' in host_spec:
Console.ok("looking for openrc")
else:
Console.error("no cm_openrc specified in the host")
return
hostname = config["cloudmesh.clouds." + host + ".cm_host"]
Console.ok("fetching information from {:} ...".format(host))
openrc = host_spec["cm_openrc"]
directory = os.path.dirname(openrc)
base = os.path.basename(openrc)
_from_dir = "{:}:{:}".format(hostname, directory).replace("~/", "")
_to_dir = os.path.dirname(Config.path_expand(directory))
openrc_file = Config.path_expand(openrc)
print("From: ", _from_dir)
print("To: ", _to_dir)
print("Openrc:", openrc_file)
cls.make_dir(_to_dir)
r = ""
Console.ok("Reading rc file from {}".format(host))
try:
r = Shell.scp('-r', _from_dir, _to_dir)
except Exception, e:
print(e)
return
def initialize(self, cloudname, user=None):
d = ConfigDict("cloudmesh.yaml")
self.cloud_details = d["cloudmesh"]["clouds"][cloudname]
# pprint(self.cloud_details)
self.cloud = cloudname
self.default_flavor = self.cloud_details["default"]["flavor"]
self.default_image = self.cloud_details["default"]["image"]
self.tenant = self.cloud_details['credentials']['OS_TENANT_NAME']
version = 2
credentials = self.cloud_details["credentials"]
cert = False
if "OS_CACERT" in credentials:
if credentials["OS_CACERT"] is not False:
cert = Config.path_expand(credentials["OS_CACERT"])
auth_url = credentials["OS_AUTH_URL"]
ksversion = auth_url.split("/")[-1]
"""
# GitHub issue 101
# mechanism to interactively ask for password
# when OS_PASSWORD set as "readline",
# or read os.environ if set as "env".
"""
os_password = credentials["OS_PASSWORD"]
prompt = "Password for cloud - {}:".format(cloudname)
if os_password.lower() in ["readline", "read", "tbd"]:
if cloudname in CloudProviderOpenstackAPI.cloud_pwd and \
'pwd' in CloudProviderOpenstackAPI.cloud_pwd[cloudname]:
os_password = CloudProviderOpenstackAPI.cloud_pwd[cloudname]["pwd"]
else:
os_password = getpass.getpass(prompt=prompt)
elif os_password.lower() == "env":
if cloudname in CloudProviderOpenstackAPI.cloud_pwd and \
'pwd' in CloudProviderOpenstackAPI.cloud_pwd[cloudname]:
os_password = CloudProviderOpenstackAPI.cloud_pwd[cloudname]["pwd"]
else:
os_password = os.environ.get("OS_PASSWORD", getpass.getpass(prompt=prompt))
# store the password for this session
CloudProviderOpenstackAPI.cloud_pwd[cloudname] = {}
CloudProviderOpenstackAPI.cloud_pwd[cloudname]["pwd"] = os_password
CloudProviderOpenstackAPI.cloud_pwd[cloudname]["status"] = "Active"
if "v2.0" == ksversion:
self.provider = client.Client(
version,
credentials["OS_USERNAME"],
os_password,
credentials["OS_TENANT_NAME"],
credentials["OS_AUTH_URL"],
cert)
elif "v3" == ksversion:
sess = session.Session(auth=self._ksv3_auth(credentials),
verify=cert)
self.provider = client.Client(2, session=sess)
def __init__(self):
self.__dict__ = self.__shared_state
if self.initialized is None:
self.filename = Config.path_expand(os.path.join("~", ".cloudmesh", "cloudmesh.db"))
self.create()
self.create_tables()
self.start()
self.user = ConfigDict(filename="cloudmesh.yaml")["cloudmesh.profile.user"]
def __init__(self):
self.__dict__ = self.__shared_state
if self.initialized is None:
self.filename = Config.path_expand(os.path.join("~", ".cloudmesh", "cloudmesh.db"))
self.create()
self.create_tables()
self.start()
self.user = ConfigDict(filename="cloudmesh.yaml")["cloudmesh.profile.user"]
def sync(cls, cloudname, localdir, remotedir, operation=None):
"""
Syncs a local directory with a remote directory.
Either from local to remote OR vice-versa
:param cloudname:
:param localdir:
:param remotedir:
:param operation: get/put
:return:
"""
# Get the operating system
os_type = cls.operating_system()
# fix the local dir path
localdirpath = Config.path_expand(localdir)
# check if local directory exists
if not os.path.exists(localdirpath):
if operation == "put":
Console.error("The local directory [{}] does not exist."
.format(localdirpath))
return None
elif operation == "get":
# for receiving, create local dir
os.mkdir(localdirpath)
Console.msg("Created local directory [{}] for sync."
.format(localdirpath))
"""
rsync now works on windows machines as well.
we install rsync (v5.4.1.20150827) on windows via chocolatey
$ choco install rsync
"""
host = cls.get_host(cloudname)
if host is None:
Console.error("Cloud [{}] not found in cloudmesh.yaml file."
.format(cloudname))
return None
else:
args = None
if operation == "put":
args = [
"-r",
localdir,
host + ":" + remotedir
]
elif operation == "get":
args = [
"-r",
host + ":" + remotedir,
localdir
]
# call rsync
return Shell.rsync(*args)
def set_os_environment(cls, cloudname):
try:
d = ConfigDict("cloudmesh.yaml")
credentials = d["cloudmesh"]["clouds"][cloudname]["credentials"]
for key in credentials.keys():
if key == "OS_CACERT":
os.environ[key] = Config.path_expand(credentials[key])
else:
os.environ[key] = credentials[key]
except Exception, e:
print(e)
def get_from_yaml(self, filename=None, load_order=None):
"""
:param filename: name of the yaml file
:return: a SSHKeyManager (dict of keys)
"""
config = None
if filename is None:
# default = Config.path_expand(os.path.join("~", ".cloudmesh", "cloudmesh.yaml"))
# config = ConfigDict("cloudmesh.yaml")
filename = "cloudmesh.yaml"
config = ConfigDict(filename)
elif load_order:
config = ConfigDict(filename, load_order)
else:
Console.error("Wrong arguments")
return
config_keys = config["cloudmesh"]["keys"]
default = config_keys["default"]
keylist = config_keys["keylist"]
sshmanager = SSHKeyManager()
for key in list(keylist.keys()):
keyname = key
value = keylist[key]
if os.path.isfile(Config.path_expand(value)):
path = Config.path_expand(value)
sshmanager.add_from_file(path, keyname)
else:
sshkey = SSHkey()
uri = Config.path_expand(os.path.join("~", ".cloudmesh", filename))
sshkey.__key__['uri'] = 'yaml://{}'.format(uri)
sshkey.__key__['string'] = value
(sshkey.__key__['type'],
sshkey.__key__['key'],
sshkey.__key__['comment']) = sshkey._parse(sshkey.__key__['string'])
sshkey.__key__['fingerprint'] = sshkey._fingerprint(sshkey.__key__['string'])
sshkey.__key__['name'] = keyname
sshkey.__key__['filename'] = filename
sshmanager.add_from_object(sshkey)
return sshmanager
"""
def set_os_environ(cls, cloudname):
"""Set os environment variables on a given cloudname"""
try:
d = ConfigDict("cloudmesh.yaml")
credentials = d["cloudmesh"]["clouds"][cloudname]["credentials"]
for key, value in credentials.iteritems():
if key == "OS_CACERT":
os.environ[key] = Config.path_expand(value)
else:
os.environ[key] = value
print("Key: " + key + ", Value: " + os.environ[key])
nova = client.Client("2",
credentials["OS_USERNAME"],
credentials["OS_PASSWORD"],
credentials["OS_TENANT_NAME"],
credentials["OS_AUTH_URL"],
Config.path_expand(credentials["OS_CACERT"]))
return nova
except Exception, e:
print(e)
def activate(self):
"""activates the shared variables"""
# engine = create_engine('sqlite:////tmp/test.db', echo=debug)
self.filename = Config.path_expand(
os.path.join("~", ".cloudmesh", "cloudmesh.db"))
self.endpoint = 'sqlite:///{:}'.format(self.filename)
self.engine = create_engine(self.endpoint)
self.Base = declarative_base(bind=self.engine)
self.meta = MetaData()
self.meta.reflect(bind=self.engine)
def activate(self):
"""activates the shared variables"""
self.debug = False
# engine = create_engine('sqlite:////tmp/test.db', echo=debug)
self.filename = Config.path_expand("~/.cloudmesh/cloudmesh.db")
self.endpoint = 'sqlite:///{:}'.format(self.filename)
self.engine = create_engine(self.endpoint)
self.Base = declarative_base(bind=self.engine)
self.meta = MetaData()
self.meta.reflect(bind=self.engine)
def set_os_environ(cloudname):
"""Set os environment variables on a given cloudname"""
# TODO: this has a severe bug as it is not unsetting variables
# Also this coded duplicates in part from register
try:
d = ConfigDict("cloudmesh.yaml")
credentials = d["cloudmesh"]["clouds"][cloudname]["credentials"]
for key, value in list(credentials.items()):
if key == "OS_CACERT":
os.environ[key] = Config.path_expand(value)
else:
os.environ[key] = value
except Exception as e:
print(e)
def set_os_environ(cloudname):
"""Set os environment variables on a given cloudname"""
# TODO: this has a severe bug as it is not unsetting variables
# Also this coded duplicates in part from register
try:
d = ConfigDict("cloudmesh.yaml")
credentials = d["cloudmesh"]["clouds"][cloudname]["credentials"]
for key, value in credentials.iteritems():
if key == "OS_CACERT":
os.environ[key] = Config.path_expand(value)
else:
os.environ[key] = value
except Exception as e:
print(e)
def add(self, key_path, keyname=None, user=None, source=None, uri=None):
"""
Adds the key to the database based on the path
:param keyname: name of the key or path to the key
:return:
"""
sshkey = SSHkey(Config.path_expand(key_path))
self.add_from_sshkey(sshkey.__key__,
keyname, user,
source=source,
uri=uri)
def add(self, key_path, keyname=None, user=None, source=None, uri=None):
"""
Adds the key to the database based on the path
:param keyname: name of the key or path to the key
:return:
"""
sshkey = SSHkey(Config.path_expand(key_path))
self.add_from_sshkey(sshkey.__key__,
keyname,
user,
source=source,
uri=uri)
def sync(cls, cloudname, localdir, remotedir, operation=None):
"""
Syncs a local directory with a remote directory.
Either from local to remote OR vice-versa
:param cloudname:
:param localdir:
:param remotedir:
:param operation: get/put
:return:
"""
# Get the operating system
os_type = cls.operating_system()
# fix the local dir path
localdirpath = Config.path_expand(localdir)
# check if local directory exists
if not os.path.exists(localdirpath):
if operation == "put":
Console.error(
"The local directory [{}] does not exist.".format(
localdirpath))
return None
elif operation == "get":
# for receiving, create local dir
os.mkdir(localdirpath)
Console.msg("Created local directory [{}] for sync.".format(
localdirpath))
"""
rsync now works on windows machines as well.
we install rsync (v5.4.1.20150827) on windows via chocolatey
$ choco install rsync
"""
host = cls.get_host(cloudname)
if host is None:
Console.error(
"Cloud [{}] not found in cloudmesh.yaml file.".format(
cloudname))
return None
else:
args = None
if operation == "put":
args = ["-r", localdir, host + ":" + remotedir]
elif operation == "get":
args = ["-r", host + ":" + remotedir, localdir]
# call rsync
return Shell.rsync(*args)
def list_ssh(cls):
"""
lists hosts from ~/.ssh/config
:return:
"""
filename = Config.path_expand("~/.ssh/config")
with open(filename, 'r') as f:
lines = f.read().split("\n")
hosts = []
for line in lines:
if "Host " in line:
host = line.strip().replace("Host ", "", 1).replace(" ", "")
hosts.append(host)
return hosts
def list_ssh(cls):
"""
lists hosts from ~/.ssh/config
:return:
"""
filename = Config.path_expand("~/.ssh/config")
with open(filename, 'r') as f:
lines = f.read().split("\n")
hosts = []
for line in lines:
if "Host " in line:
host = line.strip().replace("Host ", "", 1).replace(" ", "")
dhost = {"host": host}
hosts.append(dhost)
return hosts
def initialize(self, cloudname, user=None):
d = ConfigDict("cloudmesh.yaml")
self.cloud_details = d["cloudmesh"]["clouds"][cloudname]
# pprint(self.cloud_details)
self.cloud = cloudname
self.default_flavor = self.cloud_details["default"]["flavor"]
self.default_image = self.cloud_details["default"]["image"]
self.tenant = self.cloud_details['credentials']['OS_TENANT_NAME']
version = 2
credentials = self.cloud_details["credentials"]
cert = False
if "OS_CACERT" in credentials:
if credentials["OS_CACERT"] is not False:
cert = Config.path_expand(credentials["OS_CACERT"])
auth_url = credentials["OS_AUTH_URL"]
ksversion = auth_url.split("/")[-1]
"""
# GitHub issue 101
# mechanism to interactively ask for password
# when OS_PASSWORD set as "readline",
# or read os.environ if set as "env".
"""
os_password = credentials["OS_PASSWORD"]
if os_password.lower() == "readline":
os_password = getpass.getpass()
elif os_password.lower() == "env":
os_password = os.environ.get("OS_PASSWORD", getpass.getpass())
if "v2.0" == ksversion:
self.provider = client.Client(
version,
credentials["OS_USERNAME"],
os_password,
credentials["OS_TENANT_NAME"],
credentials["OS_AUTH_URL"],
cert)
elif "v3" == ksversion:
sess = session.Session(auth=self._ksv3_auth(credentials),
verify=cert)
self.provider = client.Client(2, session=sess)
def initialize(self, cloudname, user=None):
d = ConfigDict("cloudmesh.yaml")
self.cloud_details = d["cloudmesh"]["clouds"][cloudname]
# pprint(self.cloud_details)
self.cloud = cloudname
self.default_flavor = self.cloud_details["default"]["flavor"]
self.default_image = self.cloud_details["default"]["image"]
self.tenant = self.cloud_details['credentials']['OS_TENANT_NAME']
version = 2
credentials = self.cloud_details["credentials"]
cert = False
if "OS_CACERT" in credentials:
if credentials["OS_CACERT"] is not False:
cert = Config.path_expand(credentials["OS_CACERT"])
auth_url = credentials["OS_AUTH_URL"]
ksversion = auth_url.split("/")[-1]
"""
# GitHub issue 101
# mechanism to interactively ask for password
# when OS_PASSWORD set as "readline",
# or read os.environ if set as "env".
"""
os_password = credentials["OS_PASSWORD"]
if os_password.lower() == "readline":
os_password = getpass.getpass()
elif os_password.lower() == "env":
os_password = os.environ.get("OS_PASSWORD", getpass.getpass())
if "v2.0" == ksversion:
self.provider = client.Client(version, credentials["OS_USERNAME"],
os_password,
credentials["OS_TENANT_NAME"],
credentials["OS_AUTH_URL"], cert)
elif "v3" == ksversion:
sess = session.Session(auth=self._ksv3_auth(credentials),
verify=cert)
self.provider = client.Client(2, session=sess)
def read_rc_file(cls, host, openrc, force=False):
"""
:param host: the host name
:type host: string
:param openrc: the file name
:type openrc: string
:return:
"""
openrc = Config.path_expand(openrc)
# check if file exists
if not os.path.isfile(openrc):
Console.error("File not found.")
return
with open(openrc, 'r') as f:
lines = f.read().split("\n")
config = ConfigDict("cloudmesh.yaml")
credentials = {}
for line in lines:
if line.strip().startswith("export "):
line = line.replace("export ", "")
key, value = line.split("=", 1)
credentials[key] = value
if host not in config["cloudmesh.clouds"] or force:
config["cloudmesh"]["clouds"][host] = {
"credentials": credentials,
"cm_heading": "TBD",
"cm_host": "TBD",
"cm_label": "TBD",
"cm_type": "TBD",
"cm_type_version": "TBD"
}
config.save()
return config["cloudmesh"]["clouds"][host]["credentials"]
def read_rc_file(cls, host, openrc, force=False):
"""
:param host: the host name
:type host: string
:param openrc: the file name
:type openrc: string
:return:
"""
openrc = Config.path_expand(openrc)
# check if file exists
if not os.path.isfile(openrc):
Console.error("File not found.")
return
with open(openrc, 'r') as f:
lines = f.read().split("\n")
config = ConfigDict("cloudmesh.yaml")
credentials = {}
for line in lines:
if line.strip().startswith("export "):
line = line.replace("export ", "")
key, value = line.split("=", 1)
credentials[key] = value
if host not in config["cloudmesh.clouds"] or force:
config["cloudmesh"]["clouds"][host] = {
"credentials": credentials,
"cm_heading": "TBD",
"cm_host": "TBD",
"cm_label": "TBD",
"cm_type": "TBD",
"cm_type_version": "TBD"}
config.save()
return config["cloudmesh"]["clouds"][host]["credentials"]
def do_server(self, args, arguments):
"""
Usage:
server
Options:
-h --help
-v verbose mode
Description:
Starts up a REST service and a WEB GUI so one can browse the data in an
existing cloudmesh database.
The location of the database is supposed to be in
~/.cloud,esh/cloudmesh.db
"""
# import warnings
# with warnings.catch_warnings():
# warnings.filter("ignore")
# ignore "SQLALCHEMY_TRACK_MODIFICATIONS")
from sandman import app
from sandman.model import activate
filename = "sqlite:///{}".format(
Config.path_expand(os.path.join("~", ".cloudmesh",
"cloudmesh.db")))
print("database: {}".format(filename))
app.config['SQLALCHEMY_DATABASE_URI'] = filename
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
activate()
app.run()
def do_server(self, args, arguments):
"""
Usage:
server
Options:
-h --help
-v verbose mode
Description:
Starts up a REST service and a WEB GUI so one can browse the data in an
existing cloudmesh database.
The location of the database is supposed to be in
~/.cloud,esh/cloudmesh.db
"""
# import warnings
# with warnings.catch_warnings():
# warnings.filter("ignore")
# ignore "SQLALCHEMY_TRACK_MODIFICATIONS")
from sandman import app
from sandman.model import activate
filename = "sqlite:///{}".format(Config.path_expand(
os.path.join("~", ".cloudmesh", "cloudmesh.db")))
print("database: {}".format(filename))
app.config['SQLALCHEMY_DATABASE_URI'] = filename
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
activate()
app.run()
def __init__(self, cloudname, cm_user=None):
self.nodes = None
self.flavors = None
self.data = None
self.images = None
self.cloudname = cloudname
self.user = cm_user
OpenStack = get_driver(Provider.OPENSTACK)
self.credential = \
ConfigDict("cloudmesh.yaml")['cloudmesh']['clouds'][cloudname]['credentials']
libcloud.security.CA_CERTS_PATH = [Config.path_expand(self.credential['OS_CACERT'])]
libcloud.security.VERIFY_SSL_CERT = True
auth_url = "%s/tokens/" % self.credential['OS_AUTH_URL']
self.driver = OpenStack(
self.credential['OS_USERNAME'],
self.credential['OS_PASSWORD'],
ex_force_auth_url=auth_url,
ex_tenant_name=self.credential['OS_TENANT_NAME'],
ex_force_auth_version='2.0_password',
ex_force_service_region='regionOne')
def run(cls, cluster, group, cmd, **kwargs):
# determine the script name..
#
# TODO: script count is variable in data base, we test if fil exists and if it
# does increase counter till we find one that does not, that will be new counter.
# new counter will than be placed in db.
#
# define get_script_name(directory, prefix, counter)
# there maybe s a similar thing already in the old cloudmesh
#
# if not kwargs['-name']:
#
# old_count = Shell.ssh(cluster,
# "ls {}*.sh | wc -l | sed 's/$/ count/'".
# format(username))
# c = [f for f in old_count.splitlines() if 'count' in f]
# script_count = c[0].split()[0]
# else:
# script_count = kwargs['-name']
config = cls.read_config(cluster)
if config["credentials"]["username"] == 'TBD':
return "Please enter username in cloudmesh.yaml for cluster {}".format(cluster)
cls.incr()
data = {
"cluster": cluster,
"count": cls.counter(),
"username": config["credentials"]["username"],
"remote_experiment_dir": config["default"]["experiment_dir"],
"queue": config["default"]["queue"],
"id": None,
"nodes": 1,
"tasks_per_node": 1,
}
data["script_base_name"] = "{username}-{count}".format(**data)
data["script_name"] = "{username}-{count}.sh".format(**data)
data["script_output"] = "{username}-{count}.out".format(**data)
data["script_error"] = "{username}-{count}.err".format(**data)
data["remote_experiment_dir"] = \
"{remote_experiment_dir}/{count}".format(**data).format(**data)
data["group"] = group
# overwrite defaults
option_mapping = {'-t': '{tasks_per_node}'.format(**data),
'-N': '{nodes}'.format(**data),
'-p': '{queue}'.format(**data),
'-o': '{script_output}'.format(**data),
'-D': '{remote_experiment_dir}'.format(**data),
'-e': '{script_error}'.format(**data)}
# map(lambda k, v:
# option_mapping.__setitem__(k, kwargs.get(k) or v),
# option_mapping.items())
#
# rewrite for better readability
for (k, v) in iteritems(option_mapping):
option_mapping[k] = kwargs.get(k) or v
config = cls.read_config(cluster)
project = None
try:
project = config["credentials"]["project"]
if project.lower() not in ["tbd", "none"]:
option_mapping["-A"] = project
except:
pass
for key in option_mapping:
data[key] = option_mapping[key]
# create the options for the script
options = ""
for key, value in option_mapping.items():
options += '#SBATCH {} {}\n'.format(key, value)
cls.create_remote_dir(cluster, data["remote_experiment_dir"])
# if the command is a script, copy the script
if os.path.isfile(Config.path_expand(cmd)):
_from = Config.path_expand(cmd)
_to = '{cluster}:{remote_experiment_dir}'.format(**data)
local_file_name = cmd.split('/')[-1]
Shell.execute("rsync", [_from, _to])
data["command"] = '{remote_experiment_dir}/{local_file_name}'.format(local_file_name=local_file_name,
**data)
else:
data["command"] = cmd
data["options"] = options
script = textwrap.dedent(
"""
#! /bin/sh
{options}
echo '#CLOUDMESH: BATCH ENVIRONMENT'
echo 'BASIL_RESERVATION_ID:' $BASIL_RESERVATION_ID
echo 'SLURM_CPU_BIND:' $SLURM_CPU_BIND
echo 'SLURM_JOB_ID:' $SLURM_JOB_ID
echo 'SLURM_JOB_CPUS_PER_NODE:' $SLURM_JOB_CPUS_PER_NODE
echo 'SLURM_JOB_DEPENDENCY:' $SLURM_JOB_DEPENDENCY
echo 'SLURM_JOB_NAME:' $SLURM_JOB_NAME
echo 'SLURM_JOB_NODELIST:' $SLURM_JOB_NODELIST
echo 'SLURM_JOB_NUM_NODES:' $SLURM_JOB_NUM_NODES
echo 'SLURM_MEM_BIND:' $SLURM_MEM_BIND
echo 'SLURM_TASKS_PER_NODE:' $SLURM_TASKS_PER_NODE
echo 'MPIRUN_NOALLOCATE:' $MPIRUN_NOALLOCATE
echo 'MPIRUN_NOFREE:' $MPIRUN_NOFREE
echo 'SLURM_NTASKS_PER_CORE:' $SLURM_NTASKS_PER_CORE
echo 'SLURM_NTASKS_PER_NODE:' $SLURM_NTASKS_PER_NODE
echo 'SLURM_NTASKS_PER_SOCKET:' $SLURM_NTASKS_PER_SOCKET
echo 'SLURM_RESTART_COUNT:' $SLURM_RESTART_COUNT
echo 'SLURM_SUBMIT_DIR:' $SLURM_SUBMIT_DIR
echo 'MPIRUN_PARTITION:' $MPIRUN_PARTITION
d=$(date)
echo \"#CLOUDMESH: status, start, $d\"
srun -l echo \"#CLOUDMESH: status, start, $d\"
srun -l {command}
d=$(date)
srun -l echo \"#CLOUDMESH: status, finished, $d\"
d=$(date)
echo \"#CLOUDMESH: status, finished, $d\"
"""
).format(**data).replace("\r\n", "\n").strip()
_from = Config.path_expand('~/.cloudmesh/{script_name}'.format(**data))
_to = '{cluster}:{remote_experiment_dir}'.format(**data)
data["from"] = _from
data["to"] = _to
data["script"] = script
# write the script to local
# print(_from)
# print(_to)
with open(_from, 'w') as local_file:
local_file.write(script)
# copy to remote host
Shell.scp(_from, _to)
# delete local file
# Shell.execute('rm', _from)
# import sys; sys.exit()
# run the sbatch command
cmd = 'sbatch {remote_experiment_dir}/{script_name}'.format(**data)
data["cmd"] = cmd
# print ("CMD>", cmd)
result = Shell.ssh(cluster, cmd)
data["output"] = result
# find id
for line in result.split("\n"):
# print ("LLL>", line)
if "Submitted batch job" in line:
data["job_id"] = int(line.replace("Submitted batch job ", "").strip())
break
#
# HACK, should not depend on Model.py
#
# from cloudmesh_client.db.model import BATCHJOB
# name = ""
# BATCHJOB(name,
# cluster=data["cluster"],
# id=data["id"],
# script=data["script"]) # has user and username which seems wrong
# here what we have in data and want to store the - options are obviously wrong
# and need to be full names
# noinspection PyPep8,PyPep8
"""
{'-D': '/N/u/gvonlasz/experiment/3',
'-N': '1',
'-o': 'gvonlasz-3.out',
'-p': 'delta',
'-t': '1',
'cluster': 'india',
'cmd': 'sbatch /N/u/gvonlasz/experiment/3/gvonlasz-3.sh',
'command': 'uname',
'count': 3,
'from': '/Users/big/.cloudmesh/gvonlasz-3.sh',
'id': 1346,
'options': '#SBATCH -t 1\n#SBATCH -o gvonlasz-3.out\n#SBATCH -N 1\n#SBATCH -p delta\n#SBATCH -D /N/u/gvonlasz/experiment/3\n',
'output': 'Submitted batch job 1346',
'queue': 'delta',
'remote_experiment_dir': '/N/u/gvonlasz/experiment/3',
'script': "#! /bin/sh\n#SBATCH -t 1\n#SBATCH -o gvonlasz-3.out\n#SBATCH -N 1\n#SBATCH -p delta\n#SBATCH -D /N/u/gvonlasz/experiment/3\n\nsrun -l echo '#CLOUDMESH: Starting'\nsrun -l uname\nsrun -l echo '#CLOUDMESH: Test ok'",
'script_base_name': 'gvonlasz-3',
'script_name': 'gvonlasz-3.sh',
'script_output': 'gvonlasz-3.out',
'to': 'india:/N/u/gvonlasz/experiment/3',
'username': '******'}
"""
"""
we also want to store what part of the .out file,
BASIL_RESERVATION_ID:
SLURM_CPU_BIND:
SLURM_JOB_ID: 1351
SLURM_JOB_CPUS_PER_NODE: 12
SLURM_JOB_DEPENDENCY:
SLURM_JOB_NAME: gvonlasz-8.sh
SLURM_JOB_NODELIST: d001
SLURM_JOB_NUM_NODES: 1
SLURM_MEM_BIND:
SLURM_TASKS_PER_NODE: 12
MPIRUN_NOALLOCATE:
MPIRUN_NOFREE:
SLURM_NTASKS_PER_CORE:
SLURM_NTASKS_PER_NODE:
SLURM_NTASKS_PER_SOCKET:
SLURM_RESTART_COUNT:
SLURM_SUBMIT_DIR: /N/u/gvonlasz
MPIRUN_PARTITION:
so maybe we want to use some of the names here as they reflect the env vars
"""
#
# add data to database
#
# remove the - options
for key in ['-t', '-N', '-p', '-o', '-D', '-e']:
if key in data:
print(key, data[key])
del data[key]
data['status'] = 'started'
cls.add_db(**data)
return data
def delete(cls, name, group, cloud, force=False):
"""
deletes a VM or a set of VM
:param name: name or id of the vm to be deleted
:type name: list of strings
:param group: the group name of server
:type group: string
:param cloud: the cloud name
:type cloud: string
:param force: forces the delete process
:type force: bool
:return:
"""
# TODO: delete by group. set default cloud
# default cloud. fix this
if cloud is None:
cloud = "india"
if cloud == "india":
OpenStack = get_driver(Provider.OPENSTACK)
try:
# get cloud credential from yaml file
confd = ConfigDict("cloudmesh.yaml")
cloudcred = confd['cloudmesh']['clouds']['india'][
'credentials']
except Exception as e:
Console.error(e.message)
return
# set path to cacert and enable ssl connection
libcloud.security.CA_CERTS_PATH = [
Config.path_expand(cloudcred['OS_CACERT'])
]
libcloud.security.VERIFY_SSL_CERT = True
auth_url = "%s/tokens/" % cloudcred['OS_AUTH_URL']
driver = OpenStack(cloudcred['OS_USERNAME'],
cloudcred['OS_PASSWORD'],
ex_force_auth_url=auth_url,
ex_tenant_name=cloudcred['OS_TENANT_NAME'],
ex_force_auth_version='2.0_password',
ex_force_service_region='regionOne')
# gets all the VMs
nodes = driver.list_nodes()
def __destroy_node(node):
"""
deletes a Virtual Machine
:param node: node to be deleted
:type node: Node
:return:
"""
try:
while True:
answer = ""
if not force:
answer = raw_input(
"Would you like to delete {:}? y/n".format(
node.name))
if answer.lower() == 'y' or answer.lower(
) == 'yes' or force:
break
elif answer.lower() != 'n' and answer.lower() != 'no':
Console.ok("Invalid option")
else:
Console.ok("Operation aborted")
return
driver.destroy_node(node)
Console.ok("Virtual Machine {:} deleted".format(node.name))
except Exception as e:
Console.error(
"Could not delete Virtual Machine {:}. {:}".format(
node.name, e.message))
def __deleteNode(*args):
"""
finds a node to be deleted
:param *args: [name], [prefix], [start], [end]
:param name: full name of the vm
:type name: string
:param prefix: fist part of the vm name. Example: full name: sample-[5-12], prefix: 'sample'
:type prefix: string
:param start: first virtual machine. Example: full name: sample-[5-12], start: '5'
:type start: string
:param end: last virtual machine. Example: full name: sample-[12-99], end: 99
:type end: string
"""
if len(args) == 1: # only one vm
name = args[0]
for i in nodes:
if i.name == name:
__destroy_node(i)
return
Console.error("Virtual Machine {:} not found".format(name))
else: # interval of vms like sample-[1-10]
prefix = args[0]
start = args[1].zfill(3)
end = args[2].zfill(3)
for i in range(int(start), int(end) + 1):
name = prefix + "-" + str(i).zfill(
3) # name of the vms to be deleted
flag = False
for j in nodes:
if name == j.name:
__destroy_node(j)
flag = True
break
if not flag:
Console.error(
"Virtual Machine {:} not found".format(name))
# name is a list of strings. A string is one of:
# sample_[100-9999]. Deletes vm starting at 100 until 9999
# sample. Deletes vm named sample
for i in name:
name = i.strip()
if name[-1] == ']': # vm name like sample-[1-10]
a = (name.split('[')[1]).split(']')[0].split('-')
prefix = name.split(
'-'
)[0] # example: prefix is the sting 'sample' from sample-[10-12]
start = a[0] # type: str
end = a[1] # type: str
__deleteNode(prefix, start, end)
else: # vm name like sample-daniel
__deleteNode(name)
def start(cls, name, count, cloud, image, flavor, group):
"""
TODO: group has not been used yet. fix that
starts a virtual Machine (VM) or a set of VMs
:param name: name of the virtual machine
:type name: string, None
:param count: give the number of servers to start
:type count: integer, None
:param cloud: give a cloud to work on, if not given, selected or default cloud will be used
:type cloud: integer, None
:param image: image name
:type image: string, None
:param flavor:flavor name. m1.medium, for example
:type flavor: string, None
:param group: the group name of server
:type group: string, None
:return:
"""
# TODO: vm start (without arguments) use default cloud, image, flavor, group.
if cloud is None: # use default values for cloud, image and flavor
pass
config = CloudRegister.get(cloud)
if 'cm_type' in ["openstack"]:
provider = Provider.OPENSTACK
OpenStack = get_driver(provider)
try:
cloud_credentials = config['credentials']
except Exception as e:
Console.error(e.message)
return
# TODO: THIS MAY BE JUST TRUE IF THE CERT PATH EXISTS IN CREDENTIAL
# set path to cacert and enable ssl connection
libcloud.security.CA_CERTS_PATH = [
Config.path_expand(cloud_credentials['OS_CACERT'])
]
libcloud.security.VERIFY_SSL_CERT = True
auth_url = "%s/tokens/" % cloud_credentials['OS_AUTH_URL']
driver = OpenStack(
cloud_credentials['OS_USERNAME'],
cloud_credentials['OS_PASSWORD'],
ex_force_auth_url=auth_url,
ex_tenant_name=cloud_credentials['OS_TENANT_NAME'],
ex_force_auth_version='2.0_password',
ex_force_service_region='regionOne')
# obtain available images
# type of images: <class 'libcloud.compute.base.NodeImage'>
images = driver.list()
if not [i for i in images if i.name == image]:
Console.error("Image {:} not found".format(image))
return
image = [i for i in images if i.name == image][0]
# sizes/flavors
sizes = driver.list_sizes()
if not [i for i in sizes if i.name == flavor]:
Console.error("Flavor {:} not found".format(flavor))
return
size = [i for i in sizes if i.name == flavor][0]
if count is None:
count = 1
count = int(count)
def __findsufix():
# TODO: THIS IS A BIG BUG AS THE NEXT VM NAME IS NOT MANAGED BY SUFFIX
"""
Virtual machine name (VM) format:
string-001, string-002, ..., string-n
returns the max sufix from the VM list. It will be used in the new vm name in order to avoid
VMs with the same name.
:return: max sufix
:return type: string
"""
nodes = driver.list_nodes()
sufix = 1
for i in nodes:
n = 0
try:
n = int(
i.name.split('-', 1)[1]
) # not always is int(i.name.split('-', 1)[1] a digit
except:
pass
if sufix <= n:
sufix = n + 1
sufix = str(sufix).zfill(3)
return sufix
# set vm name
sufix = __findsufix()
global cm
if name is None:
cm.name(cloud_credentials['OS_USERNAME'] + "-" + sufix)
else:
cm.name(name + "-" + sufix)
# launch a new VM
Console.ok("Booting Virtual Machine...")
for i in range(0, count):
name = cm.get_name()
try:
node = driver.create_node(name=name,
image=image,
size=size)
except Exception as e:
Console.error(
"{:} virtual machines have not been created. {:}".
format(count - i, e.message))
return
cm.name(cm.next_name())
# wait the node to be ready before assigning public IP
sleep(10)
Console.ok("Virtual Machine created")
else:
Console.error('cloud {:} not found'.format(cloud))
d[gitkeyname]['keyname'] = keyname
d[gitkeyname]['user'] = None
d[gitkeyname]['source'] = 'git'
# sshdb.add_from_dict(d[gitkeyname])
except Exception, e:
import traceback
print(traceback.format_exc())
print (e)
Console.error("The key may already there")
elif arguments['add'] and arguments["--ssh"]:
# print('ssh add')
sshdb = SSHKeyDBManager()
keyname = arguments['--name']
filename = Config.path_expand("~/.ssh/id_rsa.pub")
try:
sshdb.add(filename, keyname, source="ssh", uri="file://" + filename)
print("Key {:} successfully added to the database".format(keyname or ""))
msg = "info. OK."
Console.ok(msg)
except Exception, e:
import traceback
print(traceback.format_exc())
print (e)
print (keyname)
print (filename)
Console.error("Problem adding the key `{}` from file `{}`".format(keyname, filename))
elif arguments['add'] and not arguments["--git"]:
from cloudmesh_client.common.ConfigDict import ConfigDict
from cloudmesh_client.common.ConfigDict import Config
from time import sleep
from pprint import pprint
from __future__ import print_function
OpenStack = get_driver(Provider.OPENSTACK)
# get cloud credential from yaml file
confd = ConfigDict("cloudmesh.yaml")
cloudcred = confd['cloudmesh']['clouds']['india']['credentials']
pprint(cloudcred)
# set path to cacert and enable ssl connection
libcloud.security.CA_CERTS_PATH = [Config.path_expand(cloudcred['OS_CACERT'])]
libcloud.security.VERIFY_SSL_CERT = True
auth_url = "%s/tokens/" % cloudcred['OS_AUTH_URL']
driver = OpenStack(cloudcred['OS_USERNAME'],
cloudcred['OS_PASSWORD'],
ex_force_auth_url=auth_url,
ex_tenant_name=cloudcred['OS_TENANT_NAME'],
ex_force_auth_version='2.0_password',
ex_force_service_region='regionOne')
# list VMs
nodes = driver.list_nodes()
print(nodes)
def _get_config_yaml_file(arguments):
filename = arguments["--yaml"] or "cloudmesh.yaml"
filename = Config.find_file(filename)
return filename
import os
from cloudmesh_client.common.ConfigDict import Config
dot_cloudmesh = os.path.join(Config.path_expand("~"), ".cloudmesh")
cloudmesh_yaml = os.path.join(Config.path_expand("~"), ".cloudmesh",
"cloudmesh.yaml")
cloudmesh_db = os.path.join(Config.path_expand("~"), ".cloudmesh",
"cloudmesh.db")
from __future__ import print_function
from cloudmesh_client.common.ConfigDict import ConfigDict
from cloudmesh_client.common.ConfigDict import Config
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base
import cloudmesh_client.common.tables as tables
from cloudmesh_client.db.models import VM, FLAVOR, DEFAULT, IMAGE
import cloudmesh_client.db.models
import cloudmesh_client.db.models as models
filename = Config.path_expand("~/test/db.db")
endpoint = 'sqlite:///{:}'.format(filename)
engine = create_engine(endpoint)
Base = declarative_base(bind=engine)
Session = sessionmaker(bind=engine)
session = Session()
r = session.query(VM).all()
for obj in r:
print(obj.name)
result = dict()
for u in r:
_id = u.id
values = {}
for key in u.__dict__.keys():
if not key.startswith("_sa"):
values[key] = u.__dict__[key]
def get_from_yaml(cls, filename=None, load_order=None, store=True):
"""
:param filename: name of the yaml file
:return: a SSHKeyManager (dict of keys)
"""
config = None
if filename is None:
# default = Config.path_expand(os.path.join("~", ".cloudmesh", "cloudmesh.yaml"))
# config = ConfigDict("cloudmesh.yaml")
filename = "cloudmesh.yaml"
config = ConfigDict(filename)
elif load_order:
config = ConfigDict(filename, load_order)
else:
Console.error("Wrong arguments")
return
config_keys = config["cloudmesh"]["keys"]
default = config_keys["default"]
keylist = config_keys["keylist"]
uri = Config.path_expand(os.path.join("~", ".cloudmesh", filename))
d = []
for key in list(keylist.keys()):
keyname = key
value = keylist[key]
if os.path.isfile(Config.path_expand(value)):
path = Config.path_expand(value)
if store:
Key.add_from_path(path, keyname)
else:
d.append(Key.add_from_path(path, keyname, store=False))
else:
keytype, string, comment = SSHkey._parse(value)
thekey = {
'uri': 'yaml://{}'.format(uri),
'string': value,
'fingerprint': SSHkey._fingerprint(value),
'name': keyname,
'comment': comment,
'source': 'git',
'kind': 'key'
}
thekey["type"], thekey["key"], thekey["comment"] = SSHkey._parse(value)
if thekey["comment"] is None:
thekey["comment"] = keyname
if store:
try:
cls.cm.add(thekey)
except:
Console.error("Key already in db", traceflag=False)
else:
d.append(thekey)
if not store:
return d
"""
def do_key(self, args, arguments):
"""
::
Usage:
key -h | --help
key list --cloud=CLOUD
key list --source=db [--format=FORMAT]
key list --source=yaml [--format=FORMAT]
key list --source=ssh [--dir=DIR] [--format=FORMAT]
key list --source=git [--format=FORMAT] [--username=USERNAME]
key list
key load [--format=FORMAT]
key add [NAME] [--source=FILENAME]
key add [NAME] [--git]
key add [NAME] [--ssh]
key get NAME
key default --select
key delete (NAME | --select | --all)
key delete NAME --cloud=CLOUD
key upload [NAME] [--cloud=CLOUD]
key upload [NAME] --active
Manages the keys
Arguments:
CLOUD The cloud
NAME The name of the key.
SOURCE db, ssh, all
KEYNAME The name of a key. For key upload it defaults to the default key name.
FORMAT The format of the output (table, json, yaml)
FILENAME The filename with full path in which the key
is located
NAME_ON_CLOUD Typically the name of the keypair on the cloud.
Options:
--dir=DIR the directory with keys [default: ~/.ssh]
--format=FORMAT the format of the output [default: table]
--source=SOURCE the source for the keys [default: db]
--username=USERNAME the source for the keys [default: none]
--name=KEYNAME The name of a key
--all delete all keys
--force delete the key form the cloud
--name_on_cloud=NAME_ON_CLOUD Typically the name of the keypair on the cloud.
Description:
key list --source=git [--username=USERNAME]
lists all keys in git for the specified user. If the
name is not specified it is read from cloudmesh.yaml
key list --source=ssh [--dir=DIR] [--format=FORMAT]
lists all keys in the directory. If the directory is not
specified the default will be ~/.ssh
key list --source=yaml [--dir=DIR] [--format=FORMAT]
lists all keys in cloudmesh.yaml file in the specified directory.
dir is by default ~/.cloudmesh
key list [--format=FORMAT]
list the keys in the giiven format: json, yaml,
table. table is default
key list
Prints list of keys. NAME of the key can be specified
key add [--name=keyname] FILENAME
adds the key specifid by the filename to the key
database
key get NAME
Retrieves the key indicated by the NAME parameter from database
and prints its fingerprint.
key default --select
Select the default key interactively
key delete NAME
deletes a key. In yaml mode it can delete only key that
are not saved in the database
key rename NAME NEW
renames the key from NAME to NEW.
"""
# pprint(arguments)
invalid_names = ['tbd', 'none', "", 'id_rsa']
def _print_dict(d, header=None, format='table'):
msg = Printer.write(
d,
order=["name", "comment", "uri", "fingerprint", "source"],
output=format,
sort_keys=True)
if msg is None:
Console.error("No keys found.", traceflag=False)
return None
else:
return msg
directory = Config.path_expand(arguments["--dir"])
cloud = arguments["--cloud"] or Default.cloud
if arguments['list']:
_format = arguments['--format']
_source = arguments['--source']
_dir = arguments['--dir']
if "--source" not in arguments and "--cloud" not in arguments:
arguments["--source"] = 'db'
if arguments['--cloud']:
#
# get key list from openstack cloud
#
#keys = Key.list(cloud, output=_format)
keys = Key.list_on_cloud(cloud, live=True, format=_format)
if keys is None:
Console.ok("The Key list is empty")
else:
print(keys)
return ""
elif arguments['--source'] == 'ssh':
try:
#sshm = SSHKeyManager()
d = Key.get_from_dir(directory, store=False)
#print("SSS", type(Key.__keys__))
#d = dict(Key.all())
#print(d)
print(
Printer.write(d,
order=[
"name", "comment", "uri",
"fingerprint", "source"
],
output="table"))
# d = dict(sshm.__keys__)
# print(_print_dict(d, format=_format))
msg = "info. OK."
Console.ok(msg)
return ""
except Exception as e:
Error.traceback(e)
Console.error("Problem listing keys from ssh")
elif arguments['--source'] in ['cm', 'cloudmesh', 'yaml']:
try:
#sshm = SSHKeyManager()
d = Key.get_from_yaml(load_order=directory, store=False)
print(
Printer.write(d,
order=[
"name", "comment", "uri",
"fingerprint", "source"
],
output=_format))
return ""
except Exception as e:
Error.traceback(e)
Console.error("Problem listing keys from `{:}`".format(
arguments['--source']))
elif arguments['--source'] in ['git']:
username = arguments["--username"]
# print(username)
if username == 'none':
conf = ConfigDict("cloudmesh.yaml")
username = conf["cloudmesh.github.username"]
#sshm = SSHKeyManager()
try:
d = Key.get_from_git(username, store=False)
print(
Printer.write(d,
order=[
"name", "comment", "uri",
"fingerprint", "source"
],
output=_format))
msg = "info. OK."
Console.ok(msg)
except Exception as e:
Error.traceback(e)
Console.error("Problem listing git keys from database")
return ""
elif arguments['--source'] == 'db':
try:
#sshdb = SSHKeyDBManager()
d = Key.all(output='dict')
if d is not None or d != []:
print(
Printer.write(d,
order=[
"name", "comment", "uri",
"fingerprint", "source"
],
output=_format))
# print(_print_dict(d, output=arguments['--format']))
msg = "info. OK."
Console.ok(msg)
else:
Console.error("No keys in the database")
except Exception as e:
Error.traceback(e)
Console.error("Problem listing keys from database")
elif arguments['get']:
try:
name = arguments['NAME']
#sshdb = SSHKeyDBManager()
d = Key.all(output="dict")
for key in d:
if key["name"] == name:
print("{:}: {:}".format(key['name'],
key['fingerprint']))
msg = "info. OK."
Console.ok(msg)
return ""
else:
pass
Console.error("The key is not in the database")
except Exception as e:
Error.traceback(e)
Console.error("The key is not in the database")
# key add [NAME] [--source=FILENAME]
# key add [NAME] [--git]
elif arguments['add'] and arguments["--git"]:
# Console.error("This feature is not yet implemented", traceflag=False)
# return ""
print('git add')
#sshdb = SSHKeyDBManager()
data = dotdict(arguments)
keyname = data.NAME
#
# get name
#
conf = ConfigDict("cloudmesh.yaml")
data.username = conf["cloudmesh.github.username"]
data.name = arguments['NAME'] or data.username
#
# get git username
#
data.username = ConfigDict(
"cloudmesh.yaml")["cloudmesh.github.username"]
if str(data.name).lower() in invalid_names:
Console.error(
"The github user name is not set in the yaml file",
traceflag=False)
return ""
try:
Console.msg(
"Retrieving github ssh keys for user {username}".format(
**data))
#sshm = SSHKeyManager()
Key.get_from_git(data.username)
d = Key.all()
# pprint(d)
except Exception as e:
Console.error(
"Problem adding keys to git for user: {username}".format(
**data))
return ""
for key in d:
if key is not None:
key["name"] = key["name"].replace("-", "_")
key["source"] = "git"
key["user"] = data.name
try:
o = dict(key)
o['value'] = key["string"]
Key.add_from_dict(key)
except Exception as e:
Console.error(
"The key {name} with that finger print already exists"
.format(**key),
traceflag=False)
elif arguments['add'] and not arguments["--git"]:
# key add [NAME] [--source=FILENAME]
# key add [NAME] [--git]
#sshdb = SSHKeyDBManager()
data = dotdict()
#
# get name
#
conf = ConfigDict("cloudmesh.yaml")
data.username = conf["cloudmesh.profile.user"]
data.name = arguments['NAME'] or data.username
data.filename = arguments['--source']
if data.filename == "db" or data.filename is None:
data.filename = Config.path_expand("~/.ssh/id_rsa.pub")
if str(data.name).lower() in invalid_names:
msg = (
"Your choice of keyname {name} is insufficient. \n"
"You must be chosing a keyname that is distingct on all clouds. \n"
"Possible choices are your gmail name, your XSEDE name, or \n"
"some name that is uniqe. "
"Best is also to set this name in \n"
"cloudmesh.profile.user as "
"part of your \n~/cloudmesh/cloudmesh.yaml file.")
Console.error(msg.format(**data), traceflag=False)
return ""
try:
Key.add_from_path(data.filename,
data.name,
source="ssh",
uri="file://" + data.filename)
print("Key {name} successfully added to the database".format(
**data))
msg = "info. OK."
Console.ok(msg)
except ValueError as e:
Console.error(
"A key with this fingerprint already exists".format(
**data),
traceflag=False)
Console.msg("Please use check with: key list")
return ""
elif arguments['default']:
# print("default")
if arguments['--select']:
keyname = None
try:
#sshdb = SSHKeyDBManager()
select = Key.select()
if select != 'q':
keyname = select.split(':')[0]
print("Setting key: {:} as default.".format(keyname))
Default.key = keyname
msg = "info. OK."
Console.ok(msg)
except Exception as e:
Error.traceback(e)
Console.error(
"Setting default for selected key {:} failed.".format(
keyname))
else:
try:
#sshdb = SSHKeyDBManager()
d = Key.table_dict()
for i in d:
if d[i]["is_default"] == "True":
key = d[i]
print("{:}: {:}".format(key['name'],
key['fingerprint']))
msg = "info. OK."
Console.ok(msg)
return ""
else:
pass
Console.error("The key is not in the database")
except Exception as e:
Error.traceback(e)
Console.error("Problem retrieving default key.")
elif arguments['delete'] and arguments["--cloud"]:
key = dotdict({
'cloud': arguments["--cloud"],
'name': arguments["NAME"]
})
try:
Key.delete(key.name, key.cloud)
msg = "info. OK."
Console.ok(msg)
except:
Console.error(
"Problem deleting the key {name} on the cloud {cloud}".
format(**key))
elif arguments['delete']:
# key delete (NAME | --select| --all)
data = dotdict({
'all': arguments['--all'] or False,
'select': arguments['--select'] or False,
'name': arguments['NAME'] or False,
})
pprint(data)
# BUG delete all is not properly implemented
if data.all:
Console.TODO("Delete --all is not yet implemented.")
# Key.delete()
elif data.select:
key = Key.select()
print(key)
else: # name
Key.delete(data.name)
msg = "info. OK."
Console.ok(msg)
elif arguments['upload']:
# pprint(arguments)
try:
#
# get username
#
conf = ConfigDict("cloudmesh.yaml")
username = conf["cloudmesh"]["profile"]["user"]
if username in ['None', 'TBD']:
username = None
#
# get cloudnames
#
clouds = []
if arguments["--active"]:
cloud = 'active'
else:
cloud = arguments["--cloud"] or Default.cloud
if cloud == "all":
config = ConfigDict("cloudmesh.yaml")
clouds = config["cloudmesh"]["clouds"]
elif cloud == 'active':
config = ConfigDict("cloudmesh.yaml")
clouds = config["cloudmesh"]["active"]
else:
clouds.append(cloud)
#
# get keyname
#
for cloud in clouds:
status = 0
#sshdb = SSHKeyDBManager()
#sshm = SSHKeyManager()
keys = Key.all()
for key in keys:
print("upload key {} -> {}".format(key["name"], cloud))
try:
status = Key.add_key_to_cloud(
username, key["name"], cloud)
except Exception as e:
Console.error("problem")
if "already exists" in str(e):
print(
"key already exists. Skipping upload. OK.")
if status == 1:
print("Problem uploading key {} to {}. failed.".
format(key["name"], cloud))
except Exception as e:
Console.error("Problem adding key to cloud")
def sync(cls, cloudname, localdir, remotedir, operation=None):
"""
Syncs a local directory with a remote directory.
Either from local to remote OR vice-versa
:param cloudname:
:param localdir:
:param remotedir:
:param operation: get/put
:return:
"""
# Get the operating system
os_type = cls.operating_system()
# fix the local dir path
localdir = Config.path_expand(localdir)
# check if local directory exists
if not os.path.exists(localdir):
if operation == "put":
Console.error("The local directory [{}] does not exist."
.format(localdir))
return None
elif operation == "get":
# for receiving, create local dir
os.mkdir(localdir)
Console.msg("Created local directory [{}] for sync."
.format(localdir))
# sync entire local directory
elif os.path.isdir(localdir):
if operation == "put":
localdir += "/*"
elif operation == "get":
localdir += "/"
# for windows use pscp
# rsync has issues with latest win10
if 'windows' in os_type:
ppk_file = ''
while ppk_file == '':
ppk_file = raw_input("Please enter putty private key(ppk) "
"file path: ")
# expand the path
ppk_file = Config.path_expand(ppk_file)
pass
host = cls.get_host(cloudname)
if host is None:
Console.error("Cloud [{}] not found in cloudmesh.yaml file."
.format(cloudname))
return None
else:
# Get the hostname and user of remote host
hostname = cls.get_hostname(host)
user = cls.get_hostuser(host)
# Console.msg("Syncing local dir [{}] with remote host [{}]"
# .format(localdir, user + "@" + hostname))
args = None
if operation == "put":
# Construct the arguments
# local dir comes first (send)
args = [
"-i",
ppk_file,
localdir,
user + "@" + hostname + ":" + remotedir
]
elif operation == "get":
# Construct the arguments
# remote dir comes first (receive)
args = [
"-i",
ppk_file,
user + "@" + hostname + ":" + remotedir,
localdir
]
try:
# Convert command to string
cmd = " ".join(["pscp"] + args)
result = os.system(cmd)
if result != 0:
Console.error("Something went wrong. Please try again later.")
return None
else:
return "Success."
except Exception as ex:
print(ex, ex.message)
# for linux/mac machines use rsync
else:
host = cls.get_host(cloudname)
if host is None:
Console.error("Cloud [{}] not found in cloudmesh.yaml file."
.format(cloudname))
return None
else:
args = None
if operation == "put":
args = [
localdir,
host + ":" + remotedir
]
elif operation == "get":
args = [
host + ":" + remotedir,
localdir
]
# call rsync
return Shell.rsync(*args)
def make_dir(cls, directory):
if not os.path.exists(Config.path_expand(directory)):
os.makedirs(Config.path_expand(directory))
from cloudmesh_client.common.ConfigDict import ConfigDict
from cloudmesh_client.common.ConfigDict import Config
from time import sleep
from pprint import pprint
from __future__ import print_function
OpenStack = get_driver(Provider.OPENSTACK)
# get cloud credential from yaml file
confd = ConfigDict("cloudmesh.yaml")
cloudcred = confd['cloudmesh']['clouds']['india']['credentials']
pprint(cloudcred)
# set path to cacert and enable ssl connection
libcloud.security.CA_CERTS_PATH = [Config.path_expand(cloudcred['OS_CACERT'])]
libcloud.security.VERIFY_SSL_CERT = True
auth_url = "%s/tokens/" % cloudcred['OS_AUTH_URL']
driver = OpenStack(cloudcred['OS_USERNAME'],
cloudcred['OS_PASSWORD'],
ex_force_auth_url=auth_url,
ex_tenant_name=cloudcred['OS_TENANT_NAME'],
ex_force_auth_version='2.0_password',
ex_force_service_region='regionOne')
# list VMs
nodes = driver.list_nodes()
print (nodes)
class CloudmeshDatabase(object):
'''
def __init__(self, user=None):
self.__dict__ = self.__shared_state
if self.initialized is None:
self.user = ConfigDict("cloudmesh.yaml")["cloudmesh.profile.user"]
self.filename = Config.path_expand(os.path.join("~", ".cloudmesh", "cloudmesh.db"))
self.engine = create_engine('sqlite:///{}'.format(self.filename), echo=False)
self.data = {"filename": self.filename}
if user is None:
self.user = ConfigDict("cloudmesh.yaml")["cloudmesh.profile.user"]
else:
self.user = user
CloudmeshDatabase.create()
CloudmeshDatabase.create_tables()
CloudmeshDatabase.start()
#
# MODEL
#
@classmethod
def create(cls):
# cls.clean()
filename = Config.path_expand(os.path.join("~", ".cloudmesh", "cloudmesh.db"))
if not os.path.isfile(filename):
cls.create_model()
'''
__shared_state = {}
data = {
"filename":
Config.path_expand(os.path.join("~", ".cloudmesh", "cloudmesh.db"))
}
initialized = None
engine = create_engine('sqlite:///{filename}'.format(**data), echo=False)
Base = declarative_base()
session = None
tables = None
# user = "******"
# user = ConfigDict("cloudmesh.yaml")["cloudmesh.profile.user"]
user = None
def __init__(self):
self.__dict__ = self.__shared_state
if self.initialized is None:
self.filename = Config.path_expand(
os.path.join("~", ".cloudmesh", "cloudmesh.db"))
self.create()
self.create_tables()
self.start()
self.user = ConfigDict(
filename="cloudmesh.yaml")["cloudmesh.profile.user"]
@classmethod
def refresh_new(cls, kind, name, **kwargs):
"""
This method refreshes the local database
with the live cloud details
:param kind:
:param name:
:param kwargs:
:return:
"""
try:
# print(cloudname)
# get the user
# TODO: Confirm user
# user = cls.user
purge = kwargs.get("purge", True)
if kind in ["flavor", "image", "vm"]:
# get provider for specific cloud
provider = CloudProvider(name).provider
current_elements = cls.find_new(category=name,
kind=kind,
output='dict',
key='name')
#returns the following:
#current_elements = {}
#for element in elements:
# current_elements[element["name"]] = element
# pprint(current_elements)
if purge:
cls.clear(kind=kind, category=name)
elements = provider.list(kind, name)
#
# image, flavor, username, group, ...
#
for element in list(elements.values()):
element["uuid"] = element['id']
element['type'] = 'string'
element["category"] = name
# element["user"] = user
element["kind"] = kind
element["provider"] = provider.cloud_type
if current_elements is not None:
for index in current_elements:
current = current_elements[index]
for attribute in [
"username", "image", "flavor", "group"
]:
if attribute in current and current[
attribute] is not None:
element[attribute] = current[attribute]
print("CCC", index, element["name"], element["flavor"])
cls.add(element)
return True
elif kind in ["batchjob"]:
# provider = BatchProvider(name).provider
# provider = BatchProvider(name)
from cloudmesh_client.cloud.hpc.BatchProvider import BatchProvider
provider = BatchProvider(name)
vms = provider.list_job(name)
for job in list(vms.values()):
job[u'uuid'] = job['id']
job[u'type'] = 'string'
job[u'category'] = name
# job[u'user'] = user
cls.add(job)
cls.save()
return True
elif kind not in ["secgroup"]:
Console.error(
"refresh not supported for this kind: {}".format(kind))
except Exception as ex:
Console.error("Problem with secgroup")
return False
@classmethod
def insert(cls, obj):
"""Insert a row into the database
:param obj: the object model to insert
:returns:
:rtype:
"""
# this method was written because I was having difficulty
# getting others to work. Not ideal, but there is a deadline
# and it is faster to write it myself than dig through the
# rest of the code to figure out how it works and how to
# deal with corner cases :(
# since some models may not be defined in the module
# db.general.model or db.openstack.model, etc, ensure that the
# DB knows about the table
if obj.__tablename__ not in cls.Base.metadata.tables.keys():
cls.create_model()
cls.session.add(obj)
cls.session.commit()
@classmethod
def select(cls, table, **filter_args):
"""Return rows of the table matching filter args.
This is a proxy for sqlalchemy's ``session.query(table).filter(**kwargs)``
:param type table: the model class
:returns: all rows in the table matching ``**filter_args``
"""
return cls.session.query(table).filter_by(**filter_args)
@classmethod
def delete_(cls, table, **filter_args):
"""Delete rows in the table matching ``filter_args``
:param type table: the model class
"""
cls.session.query(table).filter_by(**filter_args).delete()
cls.session.commit()
@classmethod
def update_(cls, table, where=None, values=None):
"""Updates a subset of rows in the table, filtered by ``where``,
setting to ``values``.
:param type table: the table class
:param dict where: match rows where all these properties hold
:param dict values: set the columns to these values
"""
cls.session.query(table)\
.filter_by(**where)\
.update(values)
cls.session.commit()
def find_new(cls, **kwargs):
"""
This method returns either
a) an array of objects from the database in dict format, that match a particular kind.
If the kind is not specified vm is used. one of the arguments must be scope="all"
b) a single entry that matches the first occurance of the query specified by kwargs,
such as name="vm_001"
:param kwargs: the arguments to be matched, scope defines if all or just the first value
is returned. first is default.
:return: a list of objects, if scope is first a single object in dotdict format is returned
"""
"""
parameters:
output="dict"
key="name"
output:
name1: element["name"]
attribute
...
name2:
....
parameters:
output="dict"
key="id"
output:
"0": index in the list of elements
attribute
...
"1":
....
parameters:
output="list"
key="id"
output:
[element0, element1, element2] each of which is a dot dict
other things
scope = "first" -> one elemnet only as dotdict (not an list)
scope = "all" -> any of the above but each element is a dotdict returns either list or dict
find -> list
to_dict(list, key="name")
to_dict(find(...), key="name") - dict of dotdicts <-- or None if we do not find
??? too complex to implement, find(...).dict(key="name") - dict of dotdicts
Default.purge = True -> when delete vm it deletes vm from db, if not keep the vm
Vm.names -> list names of all vms in db
if newvmname in Vm.names:
error vm already exists
cls.replace
cls.add -> cls.upsert
just keep add for now but introduce new upser that just calls current add?
or do refactor on add and replaces with upsert
"""
scope = kwargs.pop("scope", "all")
output = kwargs.pop("output", "dict")
table = kwargs.pop("table", None)
result = []
if table is not None:
part = cls.session.query(table).filter_by(**kwargs)
result.extend(cls.to_list(part))
else:
category = kwargs.get("category", None)
provider = kwargs.get("provider", None)
kind = kwargs.get("kind", None)
if provider is not None and kind is not None:
t = cls.table(provider, kind)
part = cls.session.query(t).filter_by(**kwargs)
if output == 'dict':
result.extend(cls.to_list(part))
else:
result.extend(part)
elif provider is None:
for t in cls.tables:
# print ("CCCC", t.__kind__, t.__provider__, kwargs)
if (t.__kind__ == kind):
part = cls.session.query(t).filter_by(**kwargs)
if output == 'dict':
result.extend(cls.to_list(part))
else:
result.extend(part)
else:
Console.error("nothing searched {}".format(kwargs))
objects = result
if len(objects) == 0:
return None
elif scope == "first":
if output == 'dict':
objects = dotdict(result[0])
else:
objects = result[0]
return objects
@classmethod
def add_new(cls, d, replace=True):
"""
o dotdict
if o is a dict an object of that type is created. It is checked if another object in the db already exists,
if so the attributes of the object will be overwritten with the once in the database
provider, kind, category, name must be set to identify the object
o is in CloudmeshDatabase.Base
this is an object of a table has been created and is to be added. It is checked if another object in the db
already exists. If so the attributes of the existing object will be updated.
"""
if d is None:
return
if type(d) in [dict, dotdict]:
if "provider" in d:
t = cls.table(kind=d["kind"], provider=d["provider"])
provider = d["provider"]
else:
t = cls.table(kind=d["kind"])
provider = t.__provider__
d["provider"] = provider
element = t(**d)
else:
element = d
if replace:
element.provider = element.__provider__
current = cls.find(provider=element.provider,
kind=element.kind,
name=element.name,
category=element.category)
if current is not None:
for key in element.__dict__.keys():
current[0][key] = element.__dict__[key]
# current[0]['user'] = element.__dict__["user"]
else:
cls.session.add(element)
else:
cls.session.add(element)
cls.save()
#
# MODEL
#
@classmethod
def create(cls):
# cls.clean()
if not os.path.isfile("{filename}".format(**cls.data)):
cls.create_model()
@classmethod
def create_model(cls):
cls.Base.metadata.create_all(cls.engine)
print("Model created")
@classmethod
def clean(cls):
for table in cls.tables:
cls.delete(kind=table.__kind__, provider=table.__provider__)
@classmethod
def create_tables(cls):
"""
:return: the list of tables in model
"""
cls.tables = [c for c in cls.Base.__subclasses__()]
@classmethod
def info(cls, kind=None):
result = []
for t in cls.tables:
entry = dict()
if kind is None or t.__kind__ in kind:
entry = {
"count": cls.session.query(t).count(),
"tablename": t.__tablename__,
"provider": t.__provider__,
"kind": t.__kind__
}
result.append(entry)
return result
@classmethod
def table(cls, provider=None, kind=None, name=None):
"""
:param category:
:param kind:
:return: the table class based on a given table name.
In case the table does not exist an exception is thrown
"""
t = None
if name is not None:
for t in cls.tables:
if t.__tablename__ == name:
return t
if provider is None and kind is not None:
t = cls.get_table_from_kind(kind)
return t
if provider is None and kind is None:
Console.error("No Kind specified")
return None
for t in cls.tables:
if t.__kind__ == kind and t.__provider__ == provider:
return t
Console.error(
"No table found for name={}, provider={}, kind={}".format(
name, provider, kind))
@classmethod
def vm_table_from_provider(cls, provider):
tablename = 'vm_{}'.format(provider)
table = cls.table(name=tablename)
return table
@classmethod
def get_table_from_kind(cls, kind):
providers = set()
for t in cls.tables:
if t.__kind__ == kind:
providers.add(t)
providers = list(providers)
if len(providers) == 1:
return providers[0]
elif len(providers) > 1:
Console.error(
"Providers for kind={} are not unique. Found={}".format(
kind, providers))
else:
Console.error("Providers for kind={} nor found".format(kind))
return None
#
# SESSION
#
# noinspection PyPep8Naming
@classmethod
def start(cls):
if cls.session is None:
# print("start session")
Session = sessionmaker(bind=cls.engine)
cls.session = Session()
@classmethod
def all(cls, provider='general', category=None, kind=None, table=None):
t = table
data = {
"provider": provider,
"kind": kind,
}
if provider is not None and kind is not None:
t = cls.table(provider=provider, kind=kind)
elif provider is None and kind is not None:
t = cls.table(kind=kind)
else:
Console.error(
"find is improperly used provider={provider} kind={kind}".
format(**data))
result = cls.session.query(t).all()
return cls.to_list(result)
@classmethod
def _find(cls,
scope='all',
provider=None,
kind=None,
output='dict',
table=None,
**kwargs):
"""
find (category="openstack", kind="vm", name="vm_002")
find (VM_OPENSTACK, kind="vm", name="vm_002") # do not use this one its only used internally
:param category:
:param kind:
:param table:
:param kwargs:
:return:
"""
t = table
if table is None:
if provider is None and kind is None:
Console.error("No provider or kind specified in find")
else:
t = cls.table(provider=provider, kind=kind)
elements = cls.session.query(t).filter_by(**kwargs)
if scope == 'first':
result = elements.first()
if result is None:
return None
if output == 'dict':
result = dotdict(cls.to_list([result])[0])
elif output == 'dict':
result = cls.to_list(elements)
elif output == 'namedict':
result = cls.to_dict(elements)
return result
@classmethod
def find(cls, **kwargs):
"""
This method returns either
a) an array of objects from the database in dict format, that match a particular kind.
If the kind is not specified vm is used. one of the arguments must be scope="all"
b) a single entry that matches the first occurance of the query specified by kwargs,
such as name="vm_001"
To select a value from a specific table:
1) identify the table of interest with :meth:`table`
>>> t = db.table(name='default')
2) specify the 'table' keywork:
>>> db.find(table=t, cm_id=42)
:param kwargs: the arguments to be matched, scope defines if all or just the first value
is returned. first is default.
:return: a list of objects, if scope is first a single object in dotdict format is returned
"""
scope = kwargs.pop("scope", "all")
output = kwargs.pop("output", "dict")
table = kwargs.pop("table", None)
result = []
if table is not None:
part = cls.session.query(table).filter_by(**kwargs)
result.extend(cls.to_list(part))
else:
category = kwargs.get("category", None)
provider = kwargs.get("provider", None)
kind = kwargs.get("kind", None)
if provider is not None and kind is not None:
t = cls.table(provider, kind)
part = cls.session.query(t).filter_by(**kwargs)
if output == 'dict':
result.extend(cls.to_list(part))
else:
result.extend(part)
elif provider is None:
for t in cls.tables:
# print ("CCCC", t.__kind__, t.__provider__, kwargs)
if (t.__kind__ == kind):
part = cls.session.query(t).filter_by(**kwargs)
if output == 'dict':
result.extend(cls.to_list(part))
else:
result.extend(part)
else:
Console.error("nothing searched {}".format(kwargs))
objects = result
'''
if len(objects) == 0 and scope == "first":
objects = None
elif len(objects) != 0:
if scope == "first":
if output == 'dict':
objects = dotdict(result[0])
else:
objects = result[0]
'''
if len(objects) == 0:
return None
elif scope == "first":
if output == 'dict':
objects = dotdict(result[0])
else:
objects = result[0]
return objects
@classmethod
def add(cls, d, replace=True):
"""
o dotdict
if o is a dict an object of that type is created. It is checked if another object in the db already exists,
if so the attributes of the object will be overwritten with the once in the database
provider, kind, category, name must be set to identify the object
o is in CloudmeshDatabase.Base
this is an object of a table has been created and is to be added. It is checked if another object in the db
already exists. If so the attributes of the existing object will be updated.
"""
if d is None:
return
if type(d) in [dict, dotdict]:
if "provider" in d:
t = cls.table(kind=d["kind"], provider=d["provider"])
provider = d["provider"]
else:
t = cls.table(kind=d["kind"])
provider = t.__provider__
d["provider"] = provider
element = t(**d)
else:
element = d
if replace:
element.provider = element.__provider__
current = cls.find(
provider=element.provider,
kind=element.kind,
name=element.name,
category=element.category,
scope=
"first" # this ensures the returned result is object/dict, not list
)
if current is not None:
for key in element.__dict__.keys():
# update based on the keys that exist in the db model
if key in current:
current[key] = element.__dict__[key]
# current['user'] = element.__dict__["user"]
# update on the db
cls.update(provider=current["provider"],
kind=current["kind"],
filter={"name": current["name"]},
update=current)
else:
cls.session.add(element)
else:
cls.session.add(element)
cls.save()
@classmethod
def add_obj(cls, objects):
for obj in list(objects.values()):
for key in list(obj.keys()):
t = cls.table(kind=key)
o = t(**obj[key])
cls.add(o)
@classmethod
def filter_by(cls, **kwargs):
"""
This method returns either
a) an array of objects from the database in dict format, that match a particular kind.
If the kind is not specified vm is used. one of the arguments must be scope="all"
b) a single entry that matches the first occurance of the query specified by kwargs,
such as name="vm_001"
:param kwargs: the arguments to be matched, scope defines if all or just the first value
is returned. first is default.
:return: a list of objects, if scope is first a single object in dotdict format is returned
"""
scope = kwargs.pop("scope", "all")
result = []
for t in cls.tables:
part = cls.session.query(t).filter_by(**kwargs)
result.extend(cls.to_list(part))
objects = result
if scope == "first" and objects is not None:
objects = dotdict(result[0])
return objects
@classmethod
def save(cls):
cls.session.commit()
cls.session.flush()
@classmethod
def to_list(cls, obj):
"""
convert the object to dict
:param obj:
:return:
"""
result = list()
for u in obj:
if u is not None:
values = {}
for key in list(u.__dict__.keys()):
if not key.startswith("_sa"):
values[key] = u.__dict__[key]
result.append(values)
return result
@classmethod
def to_dict(cls, obj, key="name"):
"""
convert the object to dict
:param obj:
:return:
"""
result = dict()
for u in obj:
if u is not None:
values = {}
for attribute in list(u.__dict__.keys()):
if not attribute.startswith("_sa"):
values[attribute] = u.__dict__[attribute]
result[values[key]] = values
return result
#
# DELETE
#
@classmethod
def delete(cls, **kwargs):
"""
:param kind:
:return:
"""
#
# BUG does not look for user related data
# user = self.user or Username()
#
result = False
provider = kwargs.get("provider", None)
kind = kwargs.get("kind")
if provider is None:
t = cls.get_table_from_kind(kind)
if provider is None or kind is None:
data = {
"provider": provider,
"kind": kind,
}
ValueError(
"find is improperly used provider={provider} kind={kind}".
format(**data))
t = cls.table(provider=provider, kind=kind)
if len(kwargs) == 0:
result = cls.session.query(t).delete()
else:
result = cls.session.query(t).filter_by(**kwargs).delete()
cls.save()
return result != 0
@classmethod
def update(cls, **kwargs):
"""
:param kind:
:param kwargs:
:return:
"""
provider = kwargs.get("provider", None)
kind = kwargs.get("kind", None)
# bug: user = self.user or Username()
if provider is not None and kind is not None:
t = cls.table(provider=provider, kind=kind)
else:
data = {
"provider": provider,
"kind": kind,
}
ValueError(
"find is improperly used provider={provider} kind={kind}".
format(**data))
filter = kwargs['filter']
values = kwargs['update']
# print (t, filter, values)
cls.session.query(t).filter_by(**filter).update(values)
cls.save()
@classmethod
def set(cls,
name,
attribute,
value,
provider=None,
kind=None,
scope="all"):
if scope == "first" and provider is None:
elements = cls.filter_by(name=name, kind=kind)[0]
# pprint(elements)
o = dotdict(elements)
# print("PPPP", kind, name, attribute, value, o)
if o[attribute] != value:
cls.update(kind=o["kind"],
provider=o["provider"],
filter={'name': name},
update={attribute: value})
elif scope == "first":
o = dotdict(
cls.filter_by(name=name, provider=provider, kind=kind)[0])
# print("PPPP", provider, kind, name, attribute, value, o)
if o[attribute] != value:
cls.update(kind=o["kind"],
provider=o["provider"],
filter={'name': name},
update={attribute: value})
elif provider is None or kind is None or scope == "all":
o = cls.filter_by(name=name)
cls.update(kind=o["kind"],
provider=o["provider"],
filter={'name': name},
update={attribute: value})
else:
Console.error("Problem setting attributes")
@classmethod
def clear(cls, kind, category, user=None):
"""
This method deletes all 'kind' entries
from the cloudmesh database
:param category: the category name
"""
# if user is None:
# user = cls.user
try:
elements = cls.find(kind=kind,
output='object',
scope="all",
category=category,
user=user)
# pprint(elements)
if elements is None:
return
for element in elements:
# pprint(element)
cls.session.delete(element)
except Exception as ex:
Console.error(ex.message)
# ###################################
# REFRESH
# ###################################
# noinspection PyUnusedLocal
@classmethod
def refresh(cls, kind, name, **kwargs):
"""
This method refreshes the local database
with the live cloud details
:param kind:
:param name:
:param kwargs:
:return:
"""
try:
# print(cloudname)
# get the user
# TODO: Confirm user
# user = cls.user
purge = kwargs.get("purge", True)
if kind in ["flavor", "image", "vm"]:
# get provider for specific cloud
provider = CloudProvider(name).provider
elements = cls.find(category=name, kind=kind, output='dict')
current_elements = {}
if elements:
for element in elements:
current_elements[element["name"]] = element
# pprint(current_elements)
# if purge:
# cls.clear(kind=kind, category=name)
elements = provider.list(kind, name)
for element in list(elements.values()):
element["uuid"] = element['id']
element['type'] = 'string'
element["category"] = name
# element["user"] = user
element["kind"] = kind
element["provider"] = provider.cloud_type
if current_elements is not None:
for index in current_elements:
current = current_elements[index]
for attribute in [
"username", "image", "flavor", "group"
]:
if attribute in current and current[
attribute] is not None:
element[attribute] = current[attribute]
# print ("CCC", index, element["name"], element["flavor"])
cls.add(element)
return True
elif kind in ["batchjob"]:
# provider = BatchProvider(name).provider
# provider = BatchProvider(name)
from cloudmesh_client.cloud.hpc.BatchProvider import BatchProvider
provider = BatchProvider(name)
vms = provider.list_job(name)
for job in list(vms.values()):
job[u'uuid'] = job['id']
job[u'type'] = 'string'
job[u'category'] = name
# job[u'user'] = user
cls.add(job)
cls.save()
return True
elif kind not in ["secgroup"]:
Console.error(
"refresh not supported for this kind: {}".format(kind))
except Exception as ex:
Console.error("Problem with secgroup")
return False
def initialize(self, cloudname, user=None):
d = ConfigDict("cloudmesh.yaml")
self.cloud_details = d["cloudmesh"]["clouds"][cloudname]
# pprint(self.cloud_details)
self.cloud = cloudname
self.default_flavor = self.cloud_details["default"]["flavor"]
self.default_image = self.cloud_details["default"]["image"]
self.tenant = self.cloud_details['credentials']['OS_TENANT_NAME']
version = 2
credentials = self.cloud_details["credentials"]
cert = False
if "OS_CACERT" in credentials:
if credentials["OS_CACERT"] is not False:
cert = Config.path_expand(credentials["OS_CACERT"])
auth_url = credentials["OS_AUTH_URL"]
ksversion = auth_url.split("/")[-1]
"""
# GitHub issue 101
# mechanism to interactively ask for password
# when OS_PASSWORD set as "readline",
# or read os.environ if set as "env".
"""
os_password = credentials["OS_PASSWORD"]
prompt = "Password for cloud - {}:".format(cloudname)
if os_password.lower() in ["readline", "read", "tbd"]:
if cloudname in CloudProviderOpenstackAPI.cloud_pwd and \
'pwd' in CloudProviderOpenstackAPI.cloud_pwd[cloudname]:
os_password = CloudProviderOpenstackAPI.cloud_pwd[cloudname][
"pwd"]
else:
os_password = getpass.getpass(prompt=prompt)
elif os_password.lower() == "env":
if cloudname in CloudProviderOpenstackAPI.cloud_pwd and \
'pwd' in CloudProviderOpenstackAPI.cloud_pwd[cloudname]:
os_password = CloudProviderOpenstackAPI.cloud_pwd[cloudname][
"pwd"]
else:
os_password = os.environ.get("OS_PASSWORD",
getpass.getpass(prompt=prompt))
# store the password for this session
CloudProviderOpenstackAPI.cloud_pwd[cloudname] = {}
CloudProviderOpenstackAPI.cloud_pwd[cloudname]["pwd"] = os_password
CloudProviderOpenstackAPI.cloud_pwd[cloudname]["status"] = "Active"
if "v2.0" == ksversion:
kws = dict(
username=credentials['OS_USERNAME'],
api_key=os_password,
project_id=credentials['OS_TENANT_NAME'],
auth_url=credentials['OS_AUTH_URL'],
)
if cert:
kws['cacert'] = cert
if not os.path.exists(cert):
Console.error(
'Certificate file `{}` not found'.format(cert))
self.provider = client.Client(version, **kws)
elif "v3" == ksversion:
sess = session.Session(auth=self._ksv3_auth(credentials),
verify=cert)
self.provider = client.Client(2, session=sess)
def do_key(self, args, arguments):
"""
::
Usage:
key -h | --help
key list [--source=db] [--format=FORMAT]
key list --source=cloudmesh [--format=FORMAT]
key list --source=ssh [--dir=DIR] [--format=FORMAT]
key list --source=git [--format=FORMAT] [--username=USERNAME]
key add --git [--name=KEYNAME] FILENAME
key add --ssh [--name=KEYNAME]
key add [--name=KEYNAME] FILENAME
key get NAME
key default [KEYNAME | --select]
key delete (KEYNAME | --select | --all) [-f]
Manages the keys
Arguments:
SOURCE db, ssh, all
KEYNAME The name of a key
FORMAT The format of the output (table, json, yaml)
FILENAME The filename with full path in which the key
is located
Options:
--dir=DIR the directory with keys [default: ~/.ssh]
--format=FORMAT the format of the output [default: table]
--source=SOURCE the source for the keys [default: db]
--username=USERNAME the source for the keys [default: none]
--name=KEYNAME The name of a key
--all delete all keys
Description:
key list --source=git [--username=USERNAME]
lists all keys in git for the specified user. If the
name is not specified it is read from cloudmesh.yaml
key list --source=ssh [--dir=DIR] [--format=FORMAT]
lists all keys in the directory. If the directory is not
specified the default will be ~/.ssh
key list --source=cloudmesh [--dir=DIR] [--format=FORMAT]
lists all keys in cloudmesh.yaml file in the specified directory.
dir is by default ~/.cloudmesh
key list [--format=FORMAT]
list the keys in teh giiven format: json, yaml,
table. table is default
key list
Prints list of keys. NAME of the key can be specified
key add [--name=keyname] FILENAME
adds the key specifid by the filename to the key
database
key get NAME
Retrieves the key indicated by the NAME parameter from database
and prints its fingerprint.
key default [NAME]
Used to set a key from the key-list as the default key
if NAME is given. Otherwise print the current default
key
key delete NAME
deletes a key. In yaml mode it can delete only key that
are not saved in the database
key rename NAME NEW
renames the key from NAME to NEW.
"""
# pprint(arguments)
def _print_dict(d, header=None, format='table'):
if format == "json":
return json.dumps(d, indent=4)
elif format == "yaml":
return yaml.dump(d, default_flow_style=False)
elif format == "table":
return dict_printer(d,
order=["name",
"comment",
"uri",
"fingerprint",
"source"],
output="table",
sort_keys=True)
else:
return d
# return dict_printer(d,order=['cm_id, name, fingerprint'])
directory = Config.path_expand(arguments["--dir"])
if arguments['list']:
_format = arguments['--format']
_source = arguments['--source']
_dir = arguments['--dir']
if arguments['--source'] == 'ssh':
try:
sshm = SSHKeyManager()
sshm.get_from_dir(directory)
d = dict(sshm.__keys__)
print(_print_dict(d, format=_format))
msg = "info. OK."
Console.ok(msg)
except Exception, e:
import traceback
print(traceback.format_exc())
print (e)
Console.error("Problem listing keys from ssh")
elif arguments['--source'] in ['cm', 'cloudmesh']:
try:
sshm = SSHKeyManager()
m = sshm.get_from_yaml(load_order=directory)
d = dict(m.__keys__)
print(_print_dict(d, format=_format))
msg = "info. OK."
Console.ok(msg)
except Exception, e:
import traceback
print(traceback.format_exc())
print (e)
Console.error("Problem listing keys from `{:}`".format(arguments['--source']))
def _get_config_yaml_file(arguments):
filename = arguments["--yaml"] or "cloudmesh.yaml"
filename = Config.find_file(filename)
return filename
def do_key(self, args, arguments):
"""
::
Usage:
key -h | --help
key list [--source=db] [--format=FORMAT]
key list --source=cloudmesh [--format=FORMAT]
key list --source=ssh [--dir=DIR] [--format=FORMAT]
key load [--format=FORMAT]
key list --source=git [--format=FORMAT] [--username=USERNAME]
key add --git [--name=KEYNAME] FILENAME
key add --ssh [--name=KEYNAME]
key add [--name=KEYNAME] FILENAME
key get NAME
key default [KEYNAME | --select]
key delete (KEYNAME | --select | --all) [--force]
key upload [KEYNAME] [--cloud=CLOUD]
key map [--cloud=CLOUD]
Manages the keys
Arguments:
SOURCE db, ssh, all
KEYNAME The name of a key. For key upload it defaults to the default key name.
FORMAT The format of the output (table, json, yaml)
FILENAME The filename with full path in which the key
is located
NAME_ON_CLOUD Typically the name of the keypair on the cloud.
Options:
--dir=DIR the directory with keys [default: ~/.ssh]
--format=FORMAT the format of the output [default: table]
--source=SOURCE the source for the keys [default: db]
--username=USERNAME the source for the keys [default: none]
--name=KEYNAME The name of a key
--all delete all keys
--force delete the key form the cloud
--name_on_cloud=NAME_ON_CLOUD Typically the name of the keypair on the cloud.
Description:
key list --source=git [--username=USERNAME]
lists all keys in git for the specified user. If the
name is not specified it is read from cloudmesh.yaml
key list --source=ssh [--dir=DIR] [--format=FORMAT]
lists all keys in the directory. If the directory is not
specified the default will be ~/.ssh
key list --source=cloudmesh [--dir=DIR] [--format=FORMAT]
lists all keys in cloudmesh.yaml file in the specified directory.
dir is by default ~/.cloudmesh
key list [--format=FORMAT]
list the keys in teh giiven format: json, yaml,
table. table is default
key list
Prints list of keys. NAME of the key can be specified
key add [--name=keyname] FILENAME
adds the key specifid by the filename to the key
database
key get NAME
Retrieves the key indicated by the NAME parameter from database
and prints its fingerprint.
key default [NAME]
Used to set a key from the key-list as the default key
if NAME is given. Otherwise print the current default
key
key delete NAME
deletes a key. In yaml mode it can delete only key that
are not saved in the database
key rename NAME NEW
renames the key from NAME to NEW.
"""
# pprint(arguments)
def _print_dict(d, header=None, format='table'):
if format == "json":
return json.dumps(d, indent=4)
elif format == "yaml":
return yaml.dump(d, default_flow_style=False)
elif format == "table":
return dict_printer(d,
order=["name",
"comment",
"uri",
"fingerprint",
"source"],
output="table",
sort_keys=True)
else:
return d
# return dict_printer(d,order=['cm_id, name, fingerprint'])
directory = Config.path_expand(arguments["--dir"])
if arguments['list']:
_format = arguments['--format']
_source = arguments['--source']
_dir = arguments['--dir']
if arguments['--source'] == 'ssh':
try:
sshm = SSHKeyManager()
sshm.get_from_dir(directory)
d = dict(sshm.__keys__)
print(_print_dict(d, format=_format))
msg = "info. OK."
Console.ok(msg)
except Exception as e:
import traceback
print(traceback.format_exc())
print (e)
Console.error("Problem listing keys from ssh")
elif arguments['--source'] in ['cm', 'cloudmesh']:
try:
sshm = SSHKeyManager()
m = sshm.get_from_yaml(load_order=directory)
d = dict(m.__keys__)
print(_print_dict(d, format=_format))
msg = "info. OK."
Console.ok(msg)
except Exception as e:
import traceback
print(traceback.format_exc())
print (e)
Console.error("Problem listing keys from `{:}`".format(arguments['--source']))
elif arguments['--source'] in ['git']:
username = arguments["--username"]
# print(username)
if username == 'none':
conf = ConfigDict("cloudmesh.yaml")
username = conf["cloudmesh.github.username"]
sshm = SSHKeyManager()
try:
sshm.get_from_git(username)
d = dict(sshm.__keys__)
print(_print_dict(d, format=_format))
msg = "info. OK."
Console.ok(msg)
except Exception as e:
import traceback
print(traceback.format_exc())
print (e)
Console.error("Problem listing git keys from database")
return ""
elif arguments['--source'] == 'db':
try:
sshdb = SSHKeyDBManager()
d = sshdb.table_dict()
if d != {}:
print(_print_dict(d, format=arguments['--format']))
msg = "info. OK."
Console.ok(msg)
else:
Console.error("No keys in the database")
except Exception as e:
import traceback
print(traceback.format_exc())
print (e)
Console.error("Problem listing keys from database")
elif arguments['load']:
_format = arguments['--format']
_dir = arguments['--dir']
try:
sshm = SSHKeyManager()
m = sshm.get_from_yaml(load_order=directory)
d = dict(m.__keys__)
sshdb = SSHKeyDBManager()
for keyname in m.__keys__:
filename = m[keyname]["path"]
try:
sshdb.add(filename,
keyname,
source="yaml",
uri="file://" + filename)
except Exception as e:
Console.error("problem adding key {}:{}".format(
keyname, filename))
print(_print_dict(d, format=_format))
msg = "info. OK."
Console.ok(msg)
except Exception as e:
Console.error("Problem adding keys from yaml file")
elif arguments['get']:
try:
name = arguments['NAME']
sshdb = SSHKeyDBManager()
d = sshdb.table_dict()
for i in d:
if d[i]["name"] == name:
key = d[i]
print("{:}: {:}".format(key['name'], key['fingerprint']))
msg = "info. OK."
Console.ok(msg)
return ""
else:
pass
Console.error("The key is not in the database")
except Exception as e:
import traceback
print(traceback.format_exc())
print (e)
Console.error("The key is not in the database")
# key add --git KEYNAME
# key add --ssh KEYNAME
# key add [--path=PATH] KEYNAME
elif arguments['add'] and arguments["--git"]:
print('git add')
sshdb = SSHKeyDBManager()
keyname = arguments['--name']
gitkeyname = arguments['NAME']
filename = arguments['FILENAME']
# Are we adding to the database as well?
# sshdb.add(filename, keyname, source="ssh", uri="file://"+filename)
username = arguments["--username"]
if username == 'none':
conf = ConfigDict("cloudmesh.yaml")
username = conf["cloudmesh.github.username"]
print(username)
sshm = SSHKeyManager()
try:
sshm.get_from_git(username)
d = dict(sshm.__keys__)
print(d)
except Exception as e:
import traceback
print(traceback.format_exc())
print (e)
Console.error("Problem adding keys to git for user: "******""
try:
# FIXME: correct code to add to git
d[gitkeyname]['keyname'] = keyname
d[gitkeyname]['user'] = None
d[gitkeyname]['source'] = 'git'
# sshdb.add_from_dict(d[gitkeyname])
except Exception as e:
Console.error("The key already exists")
elif arguments['add'] and arguments["--ssh"]:
# print('ssh add')
sshdb = SSHKeyDBManager()
keyname = arguments['--name']
filename = Config.path_expand("~/.ssh/id_rsa.pub")
try:
sshdb.add(filename, keyname, source="ssh", uri="file://" + filename)
print("Key {:} successfully added to the database".format(keyname or ""))
msg = "info. OK."
Console.ok(msg)
except Exception as e:
"""
import traceback
print(traceback.format_exc())
print (e)
print (keyname)
print (filename)
"""
Console.error("Problem adding the key `{}` from file `{}`".format(keyname, filename))
elif arguments['add'] and not arguments["--git"]:
# print('ssh add')
sshdb = SSHKeyDBManager()
keyname = arguments['--name']
filename = arguments['FILENAME']
try:
sshdb.add(filename, keyname, source="ssh", uri="file://" + filename)
print("Key {:} successfully added to the database".format(keyname or ""))
msg = "info. OK."
Console.ok(msg)
except ValueError as e:
Console.error("The key `{}` already exists".format(keyname), traceflag=False)
"""
except Exception as e:
import traceback
print(traceback.format_exc())
print (e)
print (keyname)
print (filename)
Console.error("Problem adding the key `{}` from file `{}`".format(keyname, filename))
"""
return ""
elif arguments['default']:
# print("default")
if arguments['KEYNAME']:
keyname = None
try:
keyname = arguments['KEYNAME']
sshdb = SSHKeyDBManager()
sshdb.set_default(keyname)
Default.set_key(keyname)
print("Key {:} set as default".format(keyname))
msg = "info. OK."
Console.ok(msg)
except Exception as e:
import traceback
print(traceback.format_exc())
print (e)
Console.error("Setting default for key {:} failed.".format(keyname))
elif arguments['--select']:
keyname = None
try:
sshdb = SSHKeyDBManager()
select = sshdb.select()
if select != 'q':
keyname = select.split(':')[0]
print("Setting key: {:} as default.".format(keyname))
sshdb.set_default(keyname)
msg = "info. OK."
Console.ok(msg)
except Exception as e:
import traceback
print(traceback.format_exc())
print (e)
Console.error("Setting default for selected key {:} failed.".format(keyname))
else:
try:
sshdb = SSHKeyDBManager()
d = sshdb.table_dict()
for i in d:
if d[i]["is_default"] == "True":
key = d[i]
print("{:}: {:}".format(key['name'], key['fingerprint']))
msg = "info. OK."
Console.ok(msg)
return ""
else:
pass
Console.error("The key is not in the database")
except Exception as e:
import traceback
print(traceback.format_exc())
print (e)
Console.error("Problem retrieving default key.")
elif arguments['delete']:
delete_on_cloud = arguments["--force"] or False
# print ("DDD", delete_on_cloud)
if arguments['--all']:
try:
sshm = SSHKeyManager(delete_on_cloud=delete_on_cloud)
sshm.delete_all_keys()
print("All keys from the database deleted successfully.")
msg = "info. OK."
Console.ok(msg)
except Exception as e:
import traceback
print(traceback.format_exc())
print (e)
Console.error("Problem deleting keys")
elif arguments['--select']:
keyname = None
sshdb = SSHKeyDBManager()
select = sshdb.select()
if select != 'q':
try:
keyname = select.split(':')[0]
print("Deleting key: {:}...".format(keyname))
sshm = SSHKeyManager(delete_on_cloud=delete_on_cloud)
sshm.delete_key(keyname)
msg = "info. OK."
Console.ok(msg)
except Exception as e:
import traceback
print(traceback.format_exc())
print (e)
Console.error("Problem deleting the key `{:}`".format(keyname))
else:
keyname = None
try:
keyname = arguments['KEYNAME']
sshm = SSHKeyManager(delete_on_cloud=delete_on_cloud)
sshm.delete_key(keyname)
print("Key {:} deleted successfully from database.".format(keyname))
msg = "info. OK."
Console.ok(msg)
except Exception as e:
import traceback
print(traceback.format_exc())
print (e)
Console.error("Problem deleting the key `{:}`".format(keyname))
elif arguments['upload']:
try:
#
# get username
#
conf = ConfigDict("cloudmesh.yaml")
username = conf["cloudmesh"]["profile"]["username"]
if username in ['None', 'TBD']:
username = None
#
# get cloudnames
#
clouds = []
cloud = arguments["--cloud"] or Default.get_cloud()
if cloud == "all":
config = ConfigDict("cloudmesh.yaml")
clouds = config["cloudmesh"]["clouds"]
else:
clouds.append(cloud)
#
# get keyname
#
for cloud in clouds:
status = 0
sshdb = SSHKeyDBManager()
sshm = SSHKeyManager()
keys = sshdb.find_all()
for keyid in keys:
key = keys[keyid]
print ("upload key {} -> {}".format(key["name"],
cloud))
try:
status = sshm.add_key_to_cloud(
username,
key["name"],
cloud,
key["name"])
except Exception as e:
print (e)
if "already exists" in str(e):
print ("key already exists. Skipping "
"upload. ok.")
if status == 1:
print("Problem uploading key. failed.")
msg = "info. OK."
Console.ok(msg)
except Exception as e:
import traceback
print(traceback.format_exc())
print (e)
Console.error("Problem adding key to cloud")
elif arguments['map']:
try:
cloud = arguments["--cloud"] or Default.get_cloud()
sshm = SSHKeyManager()
map_dict = sshm.get_key_cloud_maps(cloud)
print(dict_printer(map_dict,
order=["user", "key_name", "cloud_name", "key_name_on_cloud"]))
except Exception as e:
import traceback
print(traceback.format_exc())
print (e)
Console.error("Problem adding key to cloud")
def do_key(self, args, arguments):
"""
::
Usage:
key -h | --help
key list [--source=db] [--format=FORMAT]
key list --source=cloudmesh [--format=FORMAT]
key list --source=ssh [--dir=DIR] [--format=FORMAT]
key load [--format=FORMAT]
key list --source=git [--format=FORMAT] [--username=USERNAME]
key add --git [--name=KEYNAME] FILENAME
key add --ssh [--name=KEYNAME]
key add [--name=KEYNAME] FILENAME
key get NAME
key default [KEYNAME | --select]
key delete (KEYNAME | --select | --all) [--force]
key upload [KEYNAME] [--cloud=CLOUD]
key map [--cloud=CLOUD]
Manages the keys
Arguments:
SOURCE db, ssh, all
KEYNAME The name of a key. For key upload it defaults to the default key name.
FORMAT The format of the output (table, json, yaml)
FILENAME The filename with full path in which the key
is located
NAME_ON_CLOUD Typically the name of the keypair on the cloud.
Options:
--dir=DIR the directory with keys [default: ~/.ssh]
--format=FORMAT the format of the output [default: table]
--source=SOURCE the source for the keys [default: db]
--username=USERNAME the source for the keys [default: none]
--name=KEYNAME The name of a key
--all delete all keys
--force delete the key form the cloud
--name_on_cloud=NAME_ON_CLOUD Typically the name of the keypair on the cloud.
Description:
key list --source=git [--username=USERNAME]
lists all keys in git for the specified user. If the
name is not specified it is read from cloudmesh.yaml
key list --source=ssh [--dir=DIR] [--format=FORMAT]
lists all keys in the directory. If the directory is not
specified the default will be ~/.ssh
key list --source=cloudmesh [--dir=DIR] [--format=FORMAT]
lists all keys in cloudmesh.yaml file in the specified directory.
dir is by default ~/.cloudmesh
key list [--format=FORMAT]
list the keys in teh giiven format: json, yaml,
table. table is default
key list
Prints list of keys. NAME of the key can be specified
key add [--name=keyname] FILENAME
adds the key specifid by the filename to the key
database
key get NAME
Retrieves the key indicated by the NAME parameter from database
and prints its fingerprint.
key default [NAME]
Used to set a key from the key-list as the default key
if NAME is given. Otherwise print the current default
key
key delete NAME
deletes a key. In yaml mode it can delete only key that
are not saved in the database
key rename NAME NEW
renames the key from NAME to NEW.
"""
# pprint(arguments)
def _print_dict(d, header=None, format='table'):
if format == "json":
return json.dumps(d, indent=4)
elif format == "yaml":
return yaml.dump(d, default_flow_style=False)
elif format == "table":
return dict_printer(
d,
order=["name", "comment", "uri", "fingerprint", "source"],
output="table",
sort_keys=True)
else:
return d
# return dict_printer(d,order=['cm_id, name, fingerprint'])
directory = Config.path_expand(arguments["--dir"])
if arguments['list']:
_format = arguments['--format']
_source = arguments['--source']
_dir = arguments['--dir']
if arguments['--source'] == 'ssh':
try:
sshm = SSHKeyManager()
sshm.get_from_dir(directory)
d = dict(sshm.__keys__)
print(_print_dict(d, format=_format))
msg = "info. OK."
Console.ok(msg)
except Exception as e:
Error.traceback(e)
Console.error("Problem listing keys from ssh")
elif arguments['--source'] in ['cm', 'cloudmesh']:
try:
sshm = SSHKeyManager()
m = sshm.get_from_yaml(load_order=directory)
d = dict(m.__keys__)
print(_print_dict(d, format=_format))
msg = "info. OK."
Console.ok(msg)
except Exception as e:
Error.traceback(e)
Console.error("Problem listing keys from `{:}`".format(
arguments['--source']))
elif arguments['--source'] in ['git']:
username = arguments["--username"]
# print(username)
if username == 'none':
conf = ConfigDict("cloudmesh.yaml")
username = conf["cloudmesh.github.username"]
sshm = SSHKeyManager()
try:
sshm.get_from_git(username)
d = dict(sshm.__keys__)
print(_print_dict(d, format=_format))
msg = "info. OK."
Console.ok(msg)
except Exception as e:
Error.traceback(e)
Console.error("Problem listing git keys from database")
return ""
elif arguments['--source'] == 'db':
try:
sshdb = SSHKeyDBManager()
d = sshdb.table_dict()
if d != {}:
print(_print_dict(d, format=arguments['--format']))
msg = "info. OK."
Console.ok(msg)
else:
Console.error("No keys in the database")
except Exception as e:
Error.traceback(e)
Console.error("Problem listing keys from database")
elif arguments['load']:
_format = arguments['--format']
_dir = arguments['--dir']
try:
sshm = SSHKeyManager()
m = sshm.get_from_yaml(load_order=directory)
d = dict(m.__keys__)
sshdb = SSHKeyDBManager()
for keyname in m.__keys__:
filename = m[keyname]["path"]
try:
sshdb.add(filename,
keyname,
source="yaml",
uri="file://" + filename)
except Exception as e:
Console.error("problem adding key {}:{}".format(
keyname, filename))
print(_print_dict(d, format=_format))
msg = "info. OK."
Console.ok(msg)
except Exception as e:
Console.error("Problem adding keys from yaml file")
elif arguments['get']:
try:
name = arguments['NAME']
sshdb = SSHKeyDBManager()
d = sshdb.table_dict()
for i in d:
if d[i]["name"] == name:
key = d[i]
print("{:}: {:}".format(key['name'],
key['fingerprint']))
msg = "info. OK."
Console.ok(msg)
return ""
else:
pass
Console.error("The key is not in the database")
except Exception as e:
Error.traceback(e)
Console.error("The key is not in the database")
# key add --git KEYNAME
# key add --ssh KEYNAME
# key add [--path=PATH] KEYNAME
elif arguments['add'] and arguments["--git"]:
print('git add')
sshdb = SSHKeyDBManager()
keyname = arguments['--name']
gitkeyname = arguments['NAME']
filename = arguments['FILENAME']
# Are we adding to the database as well?
# sshdb.add(filename, keyname, source="ssh", uri="file://"+filename)
username = arguments["--username"]
if username == 'none':
conf = ConfigDict("cloudmesh.yaml")
username = conf["cloudmesh.github.username"]
print(username)
sshm = SSHKeyManager()
try:
sshm.get_from_git(username)
d = dict(sshm.__keys__)
print(d)
except Exception as e:
Error.traceback(e)
Console.error("Problem adding keys to git for user: "******""
try:
# FIXME: correct code to add to git
d[gitkeyname]['keyname'] = keyname
d[gitkeyname]['user'] = None
d[gitkeyname]['source'] = 'git'
# sshdb.add_from_dict(d[gitkeyname])
except Exception as e:
Console.error("The key already exists")
elif arguments['add'] and arguments["--ssh"]:
# print('ssh add')
sshdb = SSHKeyDBManager()
keyname = arguments['--name']
filename = Config.path_expand("~/.ssh/id_rsa.pub")
try:
sshdb.add(filename,
keyname,
source="ssh",
uri="file://" + filename)
print("Key {:} successfully added to the database".format(
keyname or ""))
msg = "info. OK."
Console.ok(msg)
except Exception as e:
"""
Error.traceback(e)
print (keyname)
print (filename)
"""
Console.error(
"Problem adding the key `{}` from file `{}`".format(
keyname, filename))
elif arguments['add'] and not arguments["--git"]:
# print('ssh add')
sshdb = SSHKeyDBManager()
keyname = arguments['--name']
filename = arguments['FILENAME']
try:
sshdb.add(filename,
keyname,
source="ssh",
uri="file://" + filename)
print("Key {:} successfully added to the database".format(
keyname or ""))
msg = "info. OK."
Console.ok(msg)
except ValueError as e:
Console.error("The key `{}` already exists".format(keyname),
traceflag=False)
"""
except Exception as e:
Error.traceback(e)
print (keyname)
print (filename)
Console.error("Problem adding the key `{}` from file `{}`".format(keyname, filename))
"""
return ""
elif arguments['default']:
# print("default")
if arguments['KEYNAME']:
keyname = None
try:
keyname = arguments['KEYNAME']
sshdb = SSHKeyDBManager()
sshdb.set_default(keyname)
Default.set_key(keyname)
print("Key {:} set as default".format(keyname))
msg = "info. OK."
Console.ok(msg)
except Exception as e:
Error.traceback(e)
Console.error(
"Setting default for key {:} failed.".format(keyname))
elif arguments['--select']:
keyname = None
try:
sshdb = SSHKeyDBManager()
select = sshdb.select()
if select != 'q':
keyname = select.split(':')[0]
print("Setting key: {:} as default.".format(keyname))
sshdb.set_default(keyname)
msg = "info. OK."
Console.ok(msg)
except Exception as e:
Error.traceback(e)
Console.error(
"Setting default for selected key {:} failed.".format(
keyname))
else:
try:
sshdb = SSHKeyDBManager()
d = sshdb.table_dict()
for i in d:
if d[i]["is_default"] == "True":
key = d[i]
print("{:}: {:}".format(key['name'],
key['fingerprint']))
msg = "info. OK."
Console.ok(msg)
return ""
else:
pass
Console.error("The key is not in the database")
except Exception as e:
Error.traceback(e)
Console.error("Problem retrieving default key.")
elif arguments['delete']:
delete_on_cloud = arguments["--force"] or False
# print ("DDD", delete_on_cloud)
if arguments['--all']:
try:
sshm = SSHKeyManager(delete_on_cloud=delete_on_cloud)
sshm.delete_all_keys()
print("All keys from the database deleted successfully.")
msg = "info. OK."
Console.ok(msg)
except Exception as e:
Error.traceback(e)
Console.error("Problem deleting keys")
elif arguments['--select']:
keyname = None
sshdb = SSHKeyDBManager()
select = sshdb.select()
if select != 'q':
try:
keyname = select.split(':')[0]
print("Deleting key: {:}...".format(keyname))
sshm = SSHKeyManager(delete_on_cloud=delete_on_cloud)
sshm.delete_key(keyname)
msg = "info. OK."
Console.ok(msg)
except Exception as e:
Error.traceback(e)
Console.error(
"Problem deleting the key `{:}`".format(keyname))
else:
keyname = None
try:
keyname = arguments['KEYNAME']
sshm = SSHKeyManager(delete_on_cloud=delete_on_cloud)
sshm.delete_key(keyname)
print("Key {:} deleted successfully from database.".format(
keyname))
msg = "info. OK."
Console.ok(msg)
except Exception as e:
Error.traceback(e)
Console.error(
"Problem deleting the key `{:}`".format(keyname))
elif arguments['upload']:
try:
#
# get username
#
conf = ConfigDict("cloudmesh.yaml")
username = conf["cloudmesh"]["profile"]["username"]
if username in ['None', 'TBD']:
username = None
#
# get cloudnames
#
clouds = []
cloud = arguments["--cloud"] or Default.get_cloud()
if cloud == "all":
config = ConfigDict("cloudmesh.yaml")
clouds = config["cloudmesh"]["clouds"]
else:
clouds.append(cloud)
#
# get keyname
#
for cloud in clouds:
status = 0
sshdb = SSHKeyDBManager()
sshm = SSHKeyManager()
keys = sshdb.find_all()
for keyid in keys:
key = keys[keyid]
print("upload key {} -> {}".format(key["name"], cloud))
try:
status = sshm.add_key_to_cloud(
username, key["name"], cloud, key["name"])
except Exception as e:
print(e)
if "already exists" in str(e):
print("key already exists. Skipping "
"upload. ok.")
if status == 1:
print("Problem uploading key. failed.")
msg = "info. OK."
Console.ok(msg)
except Exception as e:
Error.traceback(e)
Console.error("Problem adding key to cloud")
elif arguments['map']:
try:
cloud = arguments["--cloud"] or Default.get_cloud()
sshm = SSHKeyManager()
map_dict = sshm.get_key_cloud_maps(cloud)
print(
dict_printer(map_dict,
order=[
"user", "key_name", "cloud_name",
"key_name_on_cloud"
]))
except Exception as e:
Error.traceback(e)
Console.error("Problem adding key to cloud")
def make_dir(cls, directory):
if not os.path.exists(Config.path_expand(directory)):
os.makedirs(Config.path_expand(directory))
