def drop_database(self):
"""
dropping cloudmesh database
:return:
"""
MongoDBController().start_if_not_running()
self.client.drop_database(self.db)
def find_all_by_name(self, name, kind=None):
"""
This function returns the entry with the given name from all collections
in mongodb. The name must be unique across all collections
:param name: the unique name of the entry
:return:
"""
MongoDBController().start_if_not_running()
entries = []
if kind is None:
collections = self.collections()
else:
collections = self.collections(regex=f".*-{kind}")
for collection in collections:
try:
col = self.db[collection]
if kind is None:
cursor = col.find({"cm.name": name})
else:
cursor = col.find({"cm.name": name, "cm.kind": kind})
for entry in cursor:
entries.append(entry)
except:
pass
return entries
def collections(self, name=None, regex=None):
"""
the names of all collections
:param name: if set, only look at these collections instead of all
collections
:param regex: a regular expression on the names of the collections
:return: list of names of all collections
Example:
collections = cm.collections(regex=".*-vm")
"""
MongoDBController().start_if_not_running()
names = None
if name:
if type(name) == list:
names = name
else:
names = Parameter.expand(name)
else:
names = self.db.collection_names()
if regex:
r = re.compile(regex)
_names = list(filter(r.match, names))
names = _names
return names
def names(self, collection=None, cloud=None, kind=None, regex=None):
"""
finds all names in the specified collections. The parameters,
collection, cloud, and kind can all be Parameters that get expanded
to lists. All names from all collections are merged into the result.
With kwargs a search query on the names could be added.
Example:
cm = CmDatabase()
for kind in ['vm', "image", "flavor"]:
names = cm.names(cloud="chameleon", kind=kind)
print (names)
names = cm.names(cloud="chameleon,azure", kind="vm")
names = cm.names(collection="chameleon-image", regex="^CC-")
names = cm.names(collection="chameleon-image", regex=".*Ubuntu.*")
:param collection: The collections
:param cloud: The clouds
:param kind: The kinds
:param regex: A query applied to name
:return:
"""
MongoDBController().start_if_not_running()
collections = Parameter.expand(collection)
clouds = Parameter.expand(cloud)
kinds = Parameter.expand(kind)
result = []
def add(collection):
col = self.db[collection]
if not regex:
entries = col.find({}, {"name": 1, "_id": 0})
else:
entries = col.find({"name": {
'$regex': regex
}}, {
"name": 1,
"_id": 0
})
for entry in entries:
result.append(entry['name'])
#
# if collection is none but kind is not none find all collections
# matching THIS IS NOT YET IMPLEMENTED
#
if kinds and clouds:
for _kind in kinds:
for _cloud in clouds:
_collection = f"{_cloud}-{_kind}"
add(_collection)
if collections:
for _collection in collections:
add(_collection)
return result
def clear(self, collection="cloudmesh"):
"""
drops the collection
:return:
"""
MongoDBController().start_if_not_running()
col = self.db[collection]
col.drop()
def collection(self, name):
"""
returns the named collection
:param name: the collection name
:return: teh collection
"""
MongoDBController().start_if_not_running()
return self.db[name]
def do_stop(self, args, arguments):
"""
::
Usage:
stop
Description:
stops cloudmesh
"""
print("MongoDB stop")
if MongoDBController().service_is_running():
MongoDBController().stop()
else:
Console.ok("MongoDB service is already stopped")
def do_start(self, args, arguments):
"""
::
Usage:
start
Description:
starts cloudmesh
"""
print("MongoDB start")
if not MongoDBController().service_is_running():
MongoDBController().start(security=True)
else:
Console.ok("MongoDB service is already running")
def alter(self, entries):
MongoDBController().start_if_not_running()
# for entry in entries:
for entry in entries:
try:
# self.db["{cloud}-{kind}".format(**entry)].update(uniqueKeyVal,{'$set': keyvalToUpdate})
entry['modified'] = str(datetime.utcnow())
self.db["{cloud}-{kind}".format(**entry)].update(
{'cm': entry['cm']}, {'$set': entry})
except Exception as e:
Console.error(
"modifying document {entry}".format(entry=str(entry)))
pass
return entries
def exists(self, entries):
'''
Check if entry exists in the database
:param entries:
:return:
'''
MongoDBController().start_if_not_running()
exist_status = []
if type(entries) is dict:
entries = [entries]
for entry in entries:
collection = self.db["{cloud}-{kind}".format(**entry)]
status = collection.find(
{'cm': {'$exists': entry['cm']}}).count() > 0
exist_status.append(status)
return exist_status
def find_ok(self, collection="cloudmesh", **kwargs):
MongoDBController().start_if_not_running()
col = self.db[collection]
if len(kwargs) == 0:
entries = col.find({}, {"_id": 0})
else:
entries = col.find(kwargs, {"_id": 0})
# print ("KKKKK", kwargs)
# print ("HHHH", entries.count())
records = []
for entry in entries:
records.append(entry)
return records
def name_count(self, name):
"""
counts the occurrence of the name used in the collections
:param name: the name
:return:
"""
MongoDBController().start_if_not_running()
count = 0
collections = self.collections()
for collection in collections:
try:
entry = self.find({"cm.name": name})
count = count + len(entry)
except:
pass
return count
def find_names(self, names):
"""
Assuming names specified as parameters, it returns the entries with
these names from all collections in mongodb. The names must be unique
across all collections.
:param names: the unique names in parameter format
:return:
"""
MongoDBController().start_if_not_running()
result = []
entries = Parameter.expand(names)
if len(entries) > 0:
for entry in entries:
r = self.find_name(entry)
if r is not None:
result.append(r[0])
return result
def find_group(self, name):
"""
This function returns the entry with the given name from all collections
in mongodb. The name must be unique across all collections
:param name: the unique name of the entry
:return:
"""
MongoDBController().start_if_not_running()
entries = []
collections = self.collections()
for collection in collections:
try:
col = self.db[collection]
cursor = col.find({"cm.group": name})
for entry in cursor:
entries.append(entry)
except Exception as e:
print(e)
pass
return entries
def importAsFile(self, data, collection, db):
if collection in self.collections():
self.clear(collection=collection)
Console.msg(f"Collection {collection} dropped to be rewritten")
MongoDBController().importAsFile(data, collection, db)
def delete(self, collection="cloudmesh", **kwargs):
MongoDBController().start_if_not_running()
col = self.db[collection]
# f = col.find(kwargs)
r = col.delete_many(kwargs)
return r.deleted_count
def insert(self, d, collection="cloudmesh"):
MongoDBController().start_if_not_running()
col = self.db[collection]
col.insert_one(d)
def update(self, _entries, progress=True):
MongoDBController().start_if_not_running()
if type(_entries) == dict:
entries = [_entries]
else:
entries = _entries
if progress:
bar = Bar('Cloudmesh Database Update', max=len(entries))
result = []
for entry in entries:
if progress:
bar.next()
if 'cm' not in entry:
print("UPDATE ERROR")
VERBOSE(entry)
raise ValueError("The cm attribute is not in the entry")
entry['cm']['collection'] = "{cloud}-{kind}".format(**entry["cm"])
# noinspection PyUnusedLocal
try:
self.col = self.db[entry['cm']['collection']]
old_entry = self.col.find_one({
"cm.kind": entry["cm"]["kind"],
"cm.cloud": entry["cm"]["cloud"],
"cm.name": entry["cm"]["name"]
})
if old_entry is not None:
cm = dict(old_entry['cm'])
cm.update(entry['cm'])
cm['modified'] = str(datetime.utcnow())
# entry['cm']['created'] = cm['created']
entry['cm'] = cm
post = self.col.replace_one(
{
"cm.kind": entry['cm']["kind"],
"cm.cloud": entry['cm']["cloud"],
"cm.name": entry['cm']["name"]
},
entry,
upsert=True)
else:
entry['cm']['created'] = entry['cm']['modified'] = str(
datetime.utcnow())
self.col.insert_one(entry)
except Exception as e:
print()
Console.error(
"uploading document\n{entry}\n-------\n\n".format(
entry=str(entry)))
VERBOSE(e)
pass
result.append(entry)
if progress:
bar.finish()
return result
def do_admin(self, args, arguments):
"""
::
Usage:
admin mongo install [--brew] [--download=PATH]
admin mongo create
admin mongo status
admin mongo stats
admin mongo version
admin mongo start
admin mongo stop
admin mongo backup FILENAME
admin mongo load FILENAME
admin mongo security
admin mongo password PASSWORD
admin mongo list
admin rest status
admin rest start
admin rest stop
admin status
admin system info
admin yaml cat
admin yaml check
The admin command performs some administrative functions, such as installing packages, software and services.
It also is used to start services and configure them.
Arguments:
FILENAME the filename for backups
Options:
-f specify the file
Description:
Mongo DB
MongoDB is managed through a number of commands.
The configuration is read from ~/.cloudmesh/cloudmesh4.yaml
First, you need to create a MongoDB database with
cms admin mongo create
Second, you need to start it with
cms admin mongo start
Now you can interact with it to find out the status, the stats,
and the database listing with the commands
cms admin mongo status
cms admin mongo stats
cms admin mongo list
To stop it from running use the command
cms admin mongo stop
System information about your machine can be returned by
cms admin system info
This can be very useful in case you are filing an issue or bug.
"""
# arguments.PATH = arguments['--download'] or None
result = None
if arguments.mongo:
if arguments.install:
print("MongoDB install")
print(79 * "=")
installer = MongoInstaller()
r = installer.install()
return r
elif arguments.status:
mongo = MongoDBController()
state = mongo.status()
if "error" in state["status"]:
Console.error(state["message"])
print(Printer.attribute(state))
else:
data = dotdict()
for pid in state['output']:
entry = state['output'][pid]
data["pid"] = state['output'][pid]
data["command"] = state['output'][pid][
'command'].strip()
print(Printer.dict(data, order=["pid", "command"]))
Console.ok(str(data.pid['pid']) + " " + state["message"])
elif arguments.version:
print("MongoDB Version")
print(79 * "=")
mongo = MongoDBController()
r = mongo.version()
print(r)
elif arguments.security:
mongo = MongoDBController()
mongo.set_auth()
print()
elif arguments.create:
print("MongoDB create")
MongoDBController().create()
elif arguments.start:
print("MongoDB start")
MongoDBController().start(security=True)
elif arguments.stop:
print("MongoDB stop")
MongoDBController().stop()
elif arguments.backup:
print("MongoDB backup")
MongoDBController().dump(arguments.get('FILENAME'))
elif arguments.load:
print("MongoDB backup")
MongoDBController().restore(arguments.get('FILENAME'))
elif arguments.stats:
mongo = MongoDBController()
r = mongo.stats()
if len(r) > 0:
print(Printer.attribute(r))
Console.ok("ok")
else:
Console.ok("is your MongoDB server running")
elif arguments.list:
mongo = MongoDBController()
r = mongo.list()
if len(r) > 0:
print(
Printer.dict(r, order=["name", "sizeOnDisk", "empty"]))
Console.ok("ok")
else:
Console.ok("is your MongoDB server running")
elif arguments.yaml and arguments.cat:
path = path_expand("~/.cloudmesh/cloudmesh4.yaml")
secrets = [
"AZURE_SUBSCRIPTION_ID", "AZURE_TENANTID", "EC2_ACCESS_ID",
"EC2_SECRET_KEY", "OS_PASSWORD", "MONGO_PASSWORD"
]
with open(path) as f:
content = f.read().split("\n")
for line in content:
if "TBD" not in line:
for attribute in secrets:
if attribute + ":" in line:
line = line.split(":")[0] + ": ********"
break
print(line)
return ""
elif arguments.yaml and arguments.check:
path = path_expand("~/.cloudmesh/cloudmesh4.yaml")
print()
r = Shell.live('/Users/grey/.pyenv/shims/yamllint ' + path)
print(70 * '-')
print(" line:column description")
print()
elif arguments.rest:
if arguments.start:
print("Rest Service start")
raise NotImplementedError
elif arguments.stop:
print("Rest Service stop")
raise NotImplementedError
elif arguments.status:
print("Rest Service status")
raise NotImplementedError
elif arguments.status:
config = Config()
data = config.data["cloudmesh"]["data"]["mongo"]
# self.expanduser()
print("Rest Service status")
print("MongoDB status")
mongo = MongoDBController()
print(mongo)
# mongo.expanduser()
# data = mongo.data
# print ("DDD", data)
# data["MONGO_VERSION"] = '.'.join(str(x) for x in mongo.version())
# print (data)
# print(Printer.attribute(data))
# mongo.set_auth()
elif arguments.system:
s = OperatingSystem.get()
print(Printer.attribute(s))
return result
def find(self,
collection=None,
cloud=None,
kind=None,
query=None,
attributes=None):
"""
finds all names in the specified collections. The parameters,
collection, cloud, and kind can all be Parameters that get expanded
to lists. All names from all collections are merged into the result.
With kwargs a search query on the names could be added.
Example::
cm = CmDatabase()
for kind in ['vm', "image", "flavor"]:
entries = cm.find(cloud="chameleon", kind=kind)
print (entries)
entries = cm.find(cloud="chameleon,azure", kind="vm")
query = {"name": {'$regex': ".*Ubuntu.*"}}
entries = cm.find(collection="chameleon-image", query=query)
:param collection: The collections
:param cloud: The clouds
:param kind: The kinds
:param query: A query applied to name
:return:
"""
MongoDBController().start_if_not_running()
collections = Parameter.expand(collection)
clouds = Parameter.expand(cloud)
kinds = Parameter.expand(kind)
result = []
if not query:
query = {}
_attributes = {"_id": 0}
if attributes:
for a in attributes:
_attributes[a] = 1
def add(collection):
col = self.db[collection]
entries = col.find(query, _attributes)
for entry in entries:
result.append(entry)
if kinds and clouds:
for _kind in kinds:
for _cloud in clouds:
_collection = f"{_cloud}-{_kind}"
add(_collection)
if collections:
for _collection in collections:
add(_collection)
return result
def do_admin(self, args, arguments):
"""
::
Usage:
admin mongo install [--brew] [--download=PATH] [--nosudo] [--docker] [--dryrun] [--force]
admin mongo create
admin mongo status
admin mongo stats
admin mongo version
admin mongo start
admin mongo stop
admin mongo backup FILENAME
admin mongo load FILENAME
admin mongo security
admin mongo password PASSWORD
admin mongo list [--output=OUTPUT]
admin mongo ssh
admin status
admin system info
The admin command performs some administrative functions, such as
installing packages, software and services. It also is used to
start services and configure them.
Arguments:
FILENAME the filename for backups
Options:
-f specify the file
Description:
Mongo DB
MongoDB is managed through a number of commands.
The configuration is read from ~/.cloudmesh/cloudmesh.yaml
First, you need to create a MongoDB database with
cms admin mongo create
Second, you need to start it with
cms admin mongo start
Now you can interact with it to find out the status, the stats,
and the database listing with the commands
cms admin mongo status
cms admin mongo stats
cms admin mongo list
To stop it from running use the command
cms admin mongo stop
System information about your machine can be returned by
cms admin system info
This can be very useful in case you are filing an issue or bug.
The command
cms admin mongo ssh
is only supported for docker and allows for debugging to login
to the running container. This function may be disabled in future.
"""
map_parameters(arguments,
"output",
"nosudo",
"docker",
"dryrun",
"force")
arguments.output = arguments.output or "table"
# VERBOSE(arguments)
# arguments.PATH = arguments['--download'] or None
result = None
if arguments.mongo:
if arguments.install and arguments.docker:
installer = MongoInstaller(dryrun=arguments.dryrun,
force=arguments.force)
r = installer.docker()
return r
elif arguments.install:
print("MongoDB install")
print(79 * "=")
# print(arguments.force)
installer = MongoInstaller(dryrun=arguments.dryrun,
force=arguments.force)
sudo = not arguments.nosudo
# if 'linux' in platform.lower() :
# print("SUDO:", sudo)
# r = installer.install(sudo=sudo)
r = installer.install()
return r
elif arguments.status:
mongo = MongoDBController()
state = mongo.status()
if "error" in state["status"]:
Console.error(state["message"])
print(Printer.attribute(state))
else:
data = dotdict()
for pid in state['output']:
entry = state['output'][pid]
data["pid"] = state['output'][pid]
data["command"] = state['output'][pid][
'command'].strip()
print(Printer.dict(data, order=["pid", "command"]))
Console.ok(str(data.pid['pid']) + " " + state["message"])
elif arguments.version:
print("MongoDB Version")
print(79 * "=")
mongo = MongoDBController()
r = mongo.version()
print(r)
elif arguments.security:
mongo = MongoDBController()
mongo.set_auth()
print()
elif arguments.create:
print("MongoDB create")
MongoDBController().create()
elif arguments.ssh:
print("MongoDB ssh")
MongoDBController().ssh()
elif arguments.start:
print("MongoDB start")
MongoDBController().start(security=True)
elif arguments.stop:
print("MongoDB stop")
MongoDBController().stop()
elif arguments.backup:
print("MongoDB backup")
MongoDBController().dump(arguments.get('FILENAME'))
elif arguments.load:
print("MongoDB backup")
MongoDBController().restore(arguments.get('FILENAME'))
elif arguments.stats:
mongo = MongoDBController()
r = mongo.stats()
if len(r) > 0:
print(Printer.attribute(r))
Console.ok("ok")
else:
Console.ok("is your MongoDB server running")
elif arguments.list:
mongo = MongoDBController()
r = mongo.list()
if len(r) > 0:
if arguments.output == 'table':
print(Printer.dict(r, order=["name",
"sizeOnDisk",
"empty",
"collections"],
output=arguments.output),
)
else:
print(Printer.write(r, output=arguments.output))
Console.ok("ok")
else:
Console.ok("is your MongoDB server running")
elif arguments.status:
# config = Config()
# data = config["cloudmesh.data.mongo"]
print("Rest Service status")
print("MongoDB status")
try:
mongo = MongoDBController()
mongo.login()
if mongo.status()['status'] == 'ok':
Console.ok("Mongo is running")
except Exception as e:
Console.error("Mongo is not running")
print(e)
elif arguments.system:
s = OperatingSystem.get()
print(Printer.attribute(s))
return result
def do_init(self, args, arguments):
"""
::
Usage:
init [CLOUD] [--debug]
init yaml
Description:
Initializes cloudmesh while using data from
~/.cloudmesh/cloudmesh.yaml.
If no cloud is specified a number of local collections are
created. If a cloud is specified it also uploads the
information about images, flavors, vms. It also uploads the
security groups defined by default to the cloud.
Bug:
cms init
cms init
On Windows you have to run the cms init command twice upon
first installation
"""
if arguments.CLOUD == "yaml":
config = Config()
location = path_expand("~/.cloudmesh/cloudmesh.yaml")
path = Path(location)
if path.is_file():
print()
if yn_choice(
"The file ~/.cloudmesh/cloudmesh.yaml exists, do you wnat to overwrite it",
default='n'):
config.fetch()
print()
Console.ok("File cloudmesh.yaml downloaded from Github")
else:
print()
Console.warning("Download canceled")
print()
else:
variables = Variables()
config = Config()
try:
print("MongoDB stop")
MongoDBController().stop()
except:
Console.ok("MongoDB is not running. ok")
machine = platform.lower()
location = path_expand(config[
f'cloudmesh.data.mongo.MONGO_DOWNLOAD.{machine}.MONGO_PATH'])
try:
shutil.rmtree(location)
print("MongoDB folder deleted")
except:
Console.error(f"Could not delete {location}")
if platform == 'win32':
Console.error(f"Please try to run cms init again ... ")
exit(1)
config = Config()
user = config["cloudmesh.profile.user"]
secgroup = "flask"
print("Set key")
if user == "TBD":
Console.error(
"the user is not set in the yaml file for cloudmesh.profile.user")
sys.exit()
variables["key"] = user
Console.ok("Config Security Initialization")
Shell.execute("cms", ["config", "secinit"])
print("MongoDB create")
os.system("cms admin mongo create")
os.system("cms admin mongo start")
os.system("cms sec load")
if arguments.CLOUD is not None:
cloud = arguments.CLOUD
variables['cloud'] = cloud
os.system(f"cms key upload {user} --cloud={cloud}")
os.system(f"cms flavor list --refresh")
os.system(f"cms image list --refresh")
os.system(f"cms vm list --refresh")
os.system(f"cms sec group load {secgroup} --cloud={cloud}")
os.system(f"cms set secgroup={secgroup}")
if arguments.debug:
variables['debug'] = True
variables['timer'] = 'on'
variables['trace'] = True
variables['verbose'] = '10'
print()
print("Variables")
print()
for name in variables:
value = variables[name]
print(f" {name}={value}")
def do_init(self, args, arguments):
"""
::
Usage:
init [CLOUD] [--debug]
init yaml
Description:
Initializes cloudmesh while using data from
~/.cloudmesh/cloudmesh.yaml.
If no cloud is specified a number of local collections are
created. If a cloud is specified it also uploads the
information about images, flavors, vms. It also uploads the
security groups defined by default to the cloud.
Bug:
cms init
cms init
On Windows you have to run the cms init command twice upon
first installation
"""
ssh_key = path_expand("~/.ssh/id_rsa.pub")
if not os.path.exists(ssh_key):
Console.error(f"The ssh key {ssh_key} does not exist.")
print()
Console.info(
"cms init is a convenient program to set up cloudmesh"
" with defaukt values. Please make sure you use ssh-keygen"
" to set up the keys.\n\n"
" Additionally we recommend that you use. \n\n"
" cms test\n\n"
" to identify other issues\n")
return ""
config = Config()
if config["cloudmesh.profile.user"] == "TBD":
Console.info(
"cms init is a convenient program to set up cloudmesh"
" with defaukt values. Please make sure you use in your"
" ~/.cloudmesh/yaml file a valid value for\n\n"
" cloudmesh.profile.user\n\n"
" This name is aslo used as keyname in the cloud providers\n\n"
" Additionally we recommend that you use. \n\n"
" cms test\n\n"
" to identify other issues\n")
return ""
if arguments.CLOUD == "yaml":
location = path_expand("~/.cloudmesh/cloudmesh.yaml")
path = Path(location)
if path.is_file():
print()
if yn_choice(
"The file ~/.cloudmesh/cloudmesh.yaml exists, do you wnat to overwrite it",
default='n'):
config.fetch()
print()
Console.ok("File cloudmesh.yaml downloaded from Github")
else:
print()
Console.warning("Download canceled")
print()
else:
variables = Variables()
if config["cloudmesh.data.mongo.MODE"] != 'running':
try:
print("MongoDB stop")
MongoDBController().stop()
except:
Console.ok("MongoDB is not running. ok")
machine = platform.lower()
location = path_expand(config[
f'cloudmesh.data.mongo.MONGO_DOWNLOAD.{machine}.MONGO_PATH'])
try:
print("deleting:", location)
shutil.rmtree(location)
print("MongoDB folder deleted")
except Exception as e:
Console.error(f"Could not delete {location}")
if platform == 'win32':
print(e)
Console.error(f"Please try to run cms init again ... ")
return ""
print("MongoDB create")
os.system("cms admin mongo create")
os.system("cms admin mongo start")
else:
print("MongoDB is on \"running\" mode!")
print("Dropping cloudmesh database...")
cm_db = CmDatabase()
cm_db.connect()
cm_db.drop_database()
user = config["cloudmesh.profile.user"]
secgroup = "flask"
print("Set key")
if user == "TBD":
Console.error(
"the user is not set in the yaml file for cloudmesh.profile.user")
sys.exit()
variables["key"] = user
Console.ok("Config Security Initialization")
Shell.execute("cms", ["config", "secinit"])
os.system("cms key add")
os.system("cms sec load")
if arguments.CLOUD is not None:
cloud = arguments.CLOUD
variables['cloud'] = cloud
os.system(f"cms key upload {user} --cloud={cloud}")
os.system(f"cms flavor list --refresh")
os.system(f"cms image list --refresh")
os.system(f"cms vm list --refresh")
os.system(f"cms sec group load {secgroup} --cloud={cloud}")
os.system(f"cms set secgroup={secgroup}")
if arguments.debug:
variables['debug'] = True
variables['timer'] = 'on'
variables['trace'] = True
variables['verbose'] = '10'
print()
print("Variables")
print()
for name in variables:
value = variables[name]
print(f" {name}={value}")