def drop_database(self): """ dropping cloudmesh database :return: """ MongoDBController().start_if_not_running() self.client.drop_database(self.db)
def find_all_by_name(self, name, kind=None): """ This function returns the entry with the given name from all collections in mongodb. The name must be unique across all collections :param name: the unique name of the entry :return: """ MongoDBController().start_if_not_running() entries = [] if kind is None: collections = self.collections() else: collections = self.collections(regex=f".*-{kind}") for collection in collections: try: col = self.db[collection] if kind is None: cursor = col.find({"cm.name": name}) else: cursor = col.find({"cm.name": name, "cm.kind": kind}) for entry in cursor: entries.append(entry) except: pass return entries
def collections(self, name=None, regex=None): """ the names of all collections :param name: if set, only look at these collections instead of all collections :param regex: a regular expression on the names of the collections :return: list of names of all collections Example: collections = cm.collections(regex=".*-vm") """ MongoDBController().start_if_not_running() names = None if name: if type(name) == list: names = name else: names = Parameter.expand(name) else: names = self.db.collection_names() if regex: r = re.compile(regex) _names = list(filter(r.match, names)) names = _names return names
def names(self, collection=None, cloud=None, kind=None, regex=None): """ finds all names in the specified collections. The parameters, collection, cloud, and kind can all be Parameters that get expanded to lists. All names from all collections are merged into the result. With kwargs a search query on the names could be added. Example: cm = CmDatabase() for kind in ['vm', "image", "flavor"]: names = cm.names(cloud="chameleon", kind=kind) print (names) names = cm.names(cloud="chameleon,azure", kind="vm") names = cm.names(collection="chameleon-image", regex="^CC-") names = cm.names(collection="chameleon-image", regex=".*Ubuntu.*") :param collection: The collections :param cloud: The clouds :param kind: The kinds :param regex: A query applied to name :return: """ MongoDBController().start_if_not_running() collections = Parameter.expand(collection) clouds = Parameter.expand(cloud) kinds = Parameter.expand(kind) result = [] def add(collection): col = self.db[collection] if not regex: entries = col.find({}, {"name": 1, "_id": 0}) else: entries = col.find({"name": { '$regex': regex }}, { "name": 1, "_id": 0 }) for entry in entries: result.append(entry['name']) # # if collection is none but kind is not none find all collections # matching THIS IS NOT YET IMPLEMENTED # if kinds and clouds: for _kind in kinds: for _cloud in clouds: _collection = f"{_cloud}-{_kind}" add(_collection) if collections: for _collection in collections: add(_collection) return result
def clear(self, collection="cloudmesh"): """ drops the collection :return: """ MongoDBController().start_if_not_running() col = self.db[collection] col.drop()
def collection(self, name): """ returns the named collection :param name: the collection name :return: teh collection """ MongoDBController().start_if_not_running() return self.db[name]
def do_stop(self, args, arguments): """ :: Usage: stop Description: stops cloudmesh """ print("MongoDB stop") if MongoDBController().service_is_running(): MongoDBController().stop() else: Console.ok("MongoDB service is already stopped")
def do_start(self, args, arguments): """ :: Usage: start Description: starts cloudmesh """ print("MongoDB start") if not MongoDBController().service_is_running(): MongoDBController().start(security=True) else: Console.ok("MongoDB service is already running")
def alter(self, entries): MongoDBController().start_if_not_running() # for entry in entries: for entry in entries: try: # self.db["{cloud}-{kind}".format(**entry)].update(uniqueKeyVal,{'$set': keyvalToUpdate}) entry['modified'] = str(datetime.utcnow()) self.db["{cloud}-{kind}".format(**entry)].update( {'cm': entry['cm']}, {'$set': entry}) except Exception as e: Console.error( "modifying document {entry}".format(entry=str(entry))) pass return entries
def exists(self, entries): ''' Check if entry exists in the database :param entries: :return: ''' MongoDBController().start_if_not_running() exist_status = [] if type(entries) is dict: entries = [entries] for entry in entries: collection = self.db["{cloud}-{kind}".format(**entry)] status = collection.find( {'cm': {'$exists': entry['cm']}}).count() > 0 exist_status.append(status) return exist_status
def find_ok(self, collection="cloudmesh", **kwargs): MongoDBController().start_if_not_running() col = self.db[collection] if len(kwargs) == 0: entries = col.find({}, {"_id": 0}) else: entries = col.find(kwargs, {"_id": 0}) # print ("KKKKK", kwargs) # print ("HHHH", entries.count()) records = [] for entry in entries: records.append(entry) return records
def name_count(self, name): """ counts the occurrence of the name used in the collections :param name: the name :return: """ MongoDBController().start_if_not_running() count = 0 collections = self.collections() for collection in collections: try: entry = self.find({"cm.name": name}) count = count + len(entry) except: pass return count
def find_names(self, names): """ Assuming names specified as parameters, it returns the entries with these names from all collections in mongodb. The names must be unique across all collections. :param names: the unique names in parameter format :return: """ MongoDBController().start_if_not_running() result = [] entries = Parameter.expand(names) if len(entries) > 0: for entry in entries: r = self.find_name(entry) if r is not None: result.append(r[0]) return result
def find_group(self, name): """ This function returns the entry with the given name from all collections in mongodb. The name must be unique across all collections :param name: the unique name of the entry :return: """ MongoDBController().start_if_not_running() entries = [] collections = self.collections() for collection in collections: try: col = self.db[collection] cursor = col.find({"cm.group": name}) for entry in cursor: entries.append(entry) except Exception as e: print(e) pass return entries
def importAsFile(self, data, collection, db): if collection in self.collections(): self.clear(collection=collection) Console.msg(f"Collection {collection} dropped to be rewritten") MongoDBController().importAsFile(data, collection, db)
def delete(self, collection="cloudmesh", **kwargs): MongoDBController().start_if_not_running() col = self.db[collection] # f = col.find(kwargs) r = col.delete_many(kwargs) return r.deleted_count
def insert(self, d, collection="cloudmesh"): MongoDBController().start_if_not_running() col = self.db[collection] col.insert_one(d)
def update(self, _entries, progress=True): MongoDBController().start_if_not_running() if type(_entries) == dict: entries = [_entries] else: entries = _entries if progress: bar = Bar('Cloudmesh Database Update', max=len(entries)) result = [] for entry in entries: if progress: bar.next() if 'cm' not in entry: print("UPDATE ERROR") VERBOSE(entry) raise ValueError("The cm attribute is not in the entry") entry['cm']['collection'] = "{cloud}-{kind}".format(**entry["cm"]) # noinspection PyUnusedLocal try: self.col = self.db[entry['cm']['collection']] old_entry = self.col.find_one({ "cm.kind": entry["cm"]["kind"], "cm.cloud": entry["cm"]["cloud"], "cm.name": entry["cm"]["name"] }) if old_entry is not None: cm = dict(old_entry['cm']) cm.update(entry['cm']) cm['modified'] = str(datetime.utcnow()) # entry['cm']['created'] = cm['created'] entry['cm'] = cm post = self.col.replace_one( { "cm.kind": entry['cm']["kind"], "cm.cloud": entry['cm']["cloud"], "cm.name": entry['cm']["name"] }, entry, upsert=True) else: entry['cm']['created'] = entry['cm']['modified'] = str( datetime.utcnow()) self.col.insert_one(entry) except Exception as e: print() Console.error( "uploading document\n{entry}\n-------\n\n".format( entry=str(entry))) VERBOSE(e) pass result.append(entry) if progress: bar.finish() return result
def do_admin(self, args, arguments): """ :: Usage: admin mongo install [--brew] [--download=PATH] admin mongo create admin mongo status admin mongo stats admin mongo version admin mongo start admin mongo stop admin mongo backup FILENAME admin mongo load FILENAME admin mongo security admin mongo password PASSWORD admin mongo list admin rest status admin rest start admin rest stop admin status admin system info admin yaml cat admin yaml check The admin command performs some administrative functions, such as installing packages, software and services. It also is used to start services and configure them. Arguments: FILENAME the filename for backups Options: -f specify the file Description: Mongo DB MongoDB is managed through a number of commands. The configuration is read from ~/.cloudmesh/cloudmesh4.yaml First, you need to create a MongoDB database with cms admin mongo create Second, you need to start it with cms admin mongo start Now you can interact with it to find out the status, the stats, and the database listing with the commands cms admin mongo status cms admin mongo stats cms admin mongo list To stop it from running use the command cms admin mongo stop System information about your machine can be returned by cms admin system info This can be very useful in case you are filing an issue or bug. """ # arguments.PATH = arguments['--download'] or None result = None if arguments.mongo: if arguments.install: print("MongoDB install") print(79 * "=") installer = MongoInstaller() r = installer.install() return r elif arguments.status: mongo = MongoDBController() state = mongo.status() if "error" in state["status"]: Console.error(state["message"]) print(Printer.attribute(state)) else: data = dotdict() for pid in state['output']: entry = state['output'][pid] data["pid"] = state['output'][pid] data["command"] = state['output'][pid][ 'command'].strip() print(Printer.dict(data, order=["pid", "command"])) Console.ok(str(data.pid['pid']) + " " + state["message"]) elif arguments.version: print("MongoDB Version") print(79 * "=") mongo = MongoDBController() r = mongo.version() print(r) elif arguments.security: mongo = MongoDBController() mongo.set_auth() print() elif arguments.create: print("MongoDB create") MongoDBController().create() elif arguments.start: print("MongoDB start") MongoDBController().start(security=True) elif arguments.stop: print("MongoDB stop") MongoDBController().stop() elif arguments.backup: print("MongoDB backup") MongoDBController().dump(arguments.get('FILENAME')) elif arguments.load: print("MongoDB backup") MongoDBController().restore(arguments.get('FILENAME')) elif arguments.stats: mongo = MongoDBController() r = mongo.stats() if len(r) > 0: print(Printer.attribute(r)) Console.ok("ok") else: Console.ok("is your MongoDB server running") elif arguments.list: mongo = MongoDBController() r = mongo.list() if len(r) > 0: print( Printer.dict(r, order=["name", "sizeOnDisk", "empty"])) Console.ok("ok") else: Console.ok("is your MongoDB server running") elif arguments.yaml and arguments.cat: path = path_expand("~/.cloudmesh/cloudmesh4.yaml") secrets = [ "AZURE_SUBSCRIPTION_ID", "AZURE_TENANTID", "EC2_ACCESS_ID", "EC2_SECRET_KEY", "OS_PASSWORD", "MONGO_PASSWORD" ] with open(path) as f: content = f.read().split("\n") for line in content: if "TBD" not in line: for attribute in secrets: if attribute + ":" in line: line = line.split(":")[0] + ": ********" break print(line) return "" elif arguments.yaml and arguments.check: path = path_expand("~/.cloudmesh/cloudmesh4.yaml") print() r = Shell.live('/Users/grey/.pyenv/shims/yamllint ' + path) print(70 * '-') print(" line:column description") print() elif arguments.rest: if arguments.start: print("Rest Service start") raise NotImplementedError elif arguments.stop: print("Rest Service stop") raise NotImplementedError elif arguments.status: print("Rest Service status") raise NotImplementedError elif arguments.status: config = Config() data = config.data["cloudmesh"]["data"]["mongo"] # self.expanduser() print("Rest Service status") print("MongoDB status") mongo = MongoDBController() print(mongo) # mongo.expanduser() # data = mongo.data # print ("DDD", data) # data["MONGO_VERSION"] = '.'.join(str(x) for x in mongo.version()) # print (data) # print(Printer.attribute(data)) # mongo.set_auth() elif arguments.system: s = OperatingSystem.get() print(Printer.attribute(s)) return result
def find(self, collection=None, cloud=None, kind=None, query=None, attributes=None): """ finds all names in the specified collections. The parameters, collection, cloud, and kind can all be Parameters that get expanded to lists. All names from all collections are merged into the result. With kwargs a search query on the names could be added. Example:: cm = CmDatabase() for kind in ['vm', "image", "flavor"]: entries = cm.find(cloud="chameleon", kind=kind) print (entries) entries = cm.find(cloud="chameleon,azure", kind="vm") query = {"name": {'$regex': ".*Ubuntu.*"}} entries = cm.find(collection="chameleon-image", query=query) :param collection: The collections :param cloud: The clouds :param kind: The kinds :param query: A query applied to name :return: """ MongoDBController().start_if_not_running() collections = Parameter.expand(collection) clouds = Parameter.expand(cloud) kinds = Parameter.expand(kind) result = [] if not query: query = {} _attributes = {"_id": 0} if attributes: for a in attributes: _attributes[a] = 1 def add(collection): col = self.db[collection] entries = col.find(query, _attributes) for entry in entries: result.append(entry) if kinds and clouds: for _kind in kinds: for _cloud in clouds: _collection = f"{_cloud}-{_kind}" add(_collection) if collections: for _collection in collections: add(_collection) return result
def do_admin(self, args, arguments): """ :: Usage: admin mongo install [--brew] [--download=PATH] [--nosudo] [--docker] [--dryrun] [--force] admin mongo create admin mongo status admin mongo stats admin mongo version admin mongo start admin mongo stop admin mongo backup FILENAME admin mongo load FILENAME admin mongo security admin mongo password PASSWORD admin mongo list [--output=OUTPUT] admin mongo ssh admin status admin system info The admin command performs some administrative functions, such as installing packages, software and services. It also is used to start services and configure them. Arguments: FILENAME the filename for backups Options: -f specify the file Description: Mongo DB MongoDB is managed through a number of commands. The configuration is read from ~/.cloudmesh/cloudmesh.yaml First, you need to create a MongoDB database with cms admin mongo create Second, you need to start it with cms admin mongo start Now you can interact with it to find out the status, the stats, and the database listing with the commands cms admin mongo status cms admin mongo stats cms admin mongo list To stop it from running use the command cms admin mongo stop System information about your machine can be returned by cms admin system info This can be very useful in case you are filing an issue or bug. The command cms admin mongo ssh is only supported for docker and allows for debugging to login to the running container. This function may be disabled in future. """ map_parameters(arguments, "output", "nosudo", "docker", "dryrun", "force") arguments.output = arguments.output or "table" # VERBOSE(arguments) # arguments.PATH = arguments['--download'] or None result = None if arguments.mongo: if arguments.install and arguments.docker: installer = MongoInstaller(dryrun=arguments.dryrun, force=arguments.force) r = installer.docker() return r elif arguments.install: print("MongoDB install") print(79 * "=") # print(arguments.force) installer = MongoInstaller(dryrun=arguments.dryrun, force=arguments.force) sudo = not arguments.nosudo # if 'linux' in platform.lower() : # print("SUDO:", sudo) # r = installer.install(sudo=sudo) r = installer.install() return r elif arguments.status: mongo = MongoDBController() state = mongo.status() if "error" in state["status"]: Console.error(state["message"]) print(Printer.attribute(state)) else: data = dotdict() for pid in state['output']: entry = state['output'][pid] data["pid"] = state['output'][pid] data["command"] = state['output'][pid][ 'command'].strip() print(Printer.dict(data, order=["pid", "command"])) Console.ok(str(data.pid['pid']) + " " + state["message"]) elif arguments.version: print("MongoDB Version") print(79 * "=") mongo = MongoDBController() r = mongo.version() print(r) elif arguments.security: mongo = MongoDBController() mongo.set_auth() print() elif arguments.create: print("MongoDB create") MongoDBController().create() elif arguments.ssh: print("MongoDB ssh") MongoDBController().ssh() elif arguments.start: print("MongoDB start") MongoDBController().start(security=True) elif arguments.stop: print("MongoDB stop") MongoDBController().stop() elif arguments.backup: print("MongoDB backup") MongoDBController().dump(arguments.get('FILENAME')) elif arguments.load: print("MongoDB backup") MongoDBController().restore(arguments.get('FILENAME')) elif arguments.stats: mongo = MongoDBController() r = mongo.stats() if len(r) > 0: print(Printer.attribute(r)) Console.ok("ok") else: Console.ok("is your MongoDB server running") elif arguments.list: mongo = MongoDBController() r = mongo.list() if len(r) > 0: if arguments.output == 'table': print(Printer.dict(r, order=["name", "sizeOnDisk", "empty", "collections"], output=arguments.output), ) else: print(Printer.write(r, output=arguments.output)) Console.ok("ok") else: Console.ok("is your MongoDB server running") elif arguments.status: # config = Config() # data = config["cloudmesh.data.mongo"] print("Rest Service status") print("MongoDB status") try: mongo = MongoDBController() mongo.login() if mongo.status()['status'] == 'ok': Console.ok("Mongo is running") except Exception as e: Console.error("Mongo is not running") print(e) elif arguments.system: s = OperatingSystem.get() print(Printer.attribute(s)) return result
def do_init(self, args, arguments): """ :: Usage: init [CLOUD] [--debug] init yaml Description: Initializes cloudmesh while using data from ~/.cloudmesh/cloudmesh.yaml. If no cloud is specified a number of local collections are created. If a cloud is specified it also uploads the information about images, flavors, vms. It also uploads the security groups defined by default to the cloud. Bug: cms init cms init On Windows you have to run the cms init command twice upon first installation """ if arguments.CLOUD == "yaml": config = Config() location = path_expand("~/.cloudmesh/cloudmesh.yaml") path = Path(location) if path.is_file(): print() if yn_choice( "The file ~/.cloudmesh/cloudmesh.yaml exists, do you wnat to overwrite it", default='n'): config.fetch() print() Console.ok("File cloudmesh.yaml downloaded from Github") else: print() Console.warning("Download canceled") print() else: variables = Variables() config = Config() try: print("MongoDB stop") MongoDBController().stop() except: Console.ok("MongoDB is not running. ok") machine = platform.lower() location = path_expand(config[ f'cloudmesh.data.mongo.MONGO_DOWNLOAD.{machine}.MONGO_PATH']) try: shutil.rmtree(location) print("MongoDB folder deleted") except: Console.error(f"Could not delete {location}") if platform == 'win32': Console.error(f"Please try to run cms init again ... ") exit(1) config = Config() user = config["cloudmesh.profile.user"] secgroup = "flask" print("Set key") if user == "TBD": Console.error( "the user is not set in the yaml file for cloudmesh.profile.user") sys.exit() variables["key"] = user Console.ok("Config Security Initialization") Shell.execute("cms", ["config", "secinit"]) print("MongoDB create") os.system("cms admin mongo create") os.system("cms admin mongo start") os.system("cms sec load") if arguments.CLOUD is not None: cloud = arguments.CLOUD variables['cloud'] = cloud os.system(f"cms key upload {user} --cloud={cloud}") os.system(f"cms flavor list --refresh") os.system(f"cms image list --refresh") os.system(f"cms vm list --refresh") os.system(f"cms sec group load {secgroup} --cloud={cloud}") os.system(f"cms set secgroup={secgroup}") if arguments.debug: variables['debug'] = True variables['timer'] = 'on' variables['trace'] = True variables['verbose'] = '10' print() print("Variables") print() for name in variables: value = variables[name] print(f" {name}={value}")
def do_init(self, args, arguments): """ :: Usage: init [CLOUD] [--debug] init yaml Description: Initializes cloudmesh while using data from ~/.cloudmesh/cloudmesh.yaml. If no cloud is specified a number of local collections are created. If a cloud is specified it also uploads the information about images, flavors, vms. It also uploads the security groups defined by default to the cloud. Bug: cms init cms init On Windows you have to run the cms init command twice upon first installation """ ssh_key = path_expand("~/.ssh/id_rsa.pub") if not os.path.exists(ssh_key): Console.error(f"The ssh key {ssh_key} does not exist.") print() Console.info( "cms init is a convenient program to set up cloudmesh" " with defaukt values. Please make sure you use ssh-keygen" " to set up the keys.\n\n" " Additionally we recommend that you use. \n\n" " cms test\n\n" " to identify other issues\n") return "" config = Config() if config["cloudmesh.profile.user"] == "TBD": Console.info( "cms init is a convenient program to set up cloudmesh" " with defaukt values. Please make sure you use in your" " ~/.cloudmesh/yaml file a valid value for\n\n" " cloudmesh.profile.user\n\n" " This name is aslo used as keyname in the cloud providers\n\n" " Additionally we recommend that you use. \n\n" " cms test\n\n" " to identify other issues\n") return "" if arguments.CLOUD == "yaml": location = path_expand("~/.cloudmesh/cloudmesh.yaml") path = Path(location) if path.is_file(): print() if yn_choice( "The file ~/.cloudmesh/cloudmesh.yaml exists, do you wnat to overwrite it", default='n'): config.fetch() print() Console.ok("File cloudmesh.yaml downloaded from Github") else: print() Console.warning("Download canceled") print() else: variables = Variables() if config["cloudmesh.data.mongo.MODE"] != 'running': try: print("MongoDB stop") MongoDBController().stop() except: Console.ok("MongoDB is not running. ok") machine = platform.lower() location = path_expand(config[ f'cloudmesh.data.mongo.MONGO_DOWNLOAD.{machine}.MONGO_PATH']) try: print("deleting:", location) shutil.rmtree(location) print("MongoDB folder deleted") except Exception as e: Console.error(f"Could not delete {location}") if platform == 'win32': print(e) Console.error(f"Please try to run cms init again ... ") return "" print("MongoDB create") os.system("cms admin mongo create") os.system("cms admin mongo start") else: print("MongoDB is on \"running\" mode!") print("Dropping cloudmesh database...") cm_db = CmDatabase() cm_db.connect() cm_db.drop_database() user = config["cloudmesh.profile.user"] secgroup = "flask" print("Set key") if user == "TBD": Console.error( "the user is not set in the yaml file for cloudmesh.profile.user") sys.exit() variables["key"] = user Console.ok("Config Security Initialization") Shell.execute("cms", ["config", "secinit"]) os.system("cms key add") os.system("cms sec load") if arguments.CLOUD is not None: cloud = arguments.CLOUD variables['cloud'] = cloud os.system(f"cms key upload {user} --cloud={cloud}") os.system(f"cms flavor list --refresh") os.system(f"cms image list --refresh") os.system(f"cms vm list --refresh") os.system(f"cms sec group load {secgroup} --cloud={cloud}") os.system(f"cms set secgroup={secgroup}") if arguments.debug: variables['debug'] = True variables['timer'] = 'on' variables['trace'] = True variables['verbose'] = '10' print() print("Variables") print() for name in variables: value = variables[name] print(f" {name}={value}")