def pbs_service(self, rack_name=None): config = cm_config() username = config.get("cloudmesh.hpc.username") pbs = PBS(username, "india.futuregrid.org") dict_pbs_info = pbs.pbsnodes() dict_data = {} inventory = Inventory() racks = inventory.get_clusters() for rack in racks: rack_name = rack["cm_cluster"] dict_data[rack_name] = {} hosts = rack["cm_value"] for host in hosts: (hid, hlabel) = inventory.get_host_id_label(host, "public") utype = "unknown" if hlabel in dict_pbs_info: server = dict_pbs_info[hlabel] if "note" in server.keys(): note_value = server["note"] # to compatible with the future change if type(note_value) is dict: utype = note_value["service"] else: # currently is a literal string for note utype = note_value dict_data[rack_name][hid] = utype return dict_data
def setup(self): self.cluster = "bravo" self.name = "b010" self.inventory = Inventory() self.inventory.clear() self.inventory.generate() print("GENERATION COMPLETE")
def activate_cm_shell_inventory(self): self.register_command_topic('cloud', 'inventory') # # BUG this needs to be done not in activate # self.inventory_name = "test" # port number is missing # should be imported from cloudmesh_server.yaml db = connect(self.inventory_name) self.inventory = Inventory() pass
def setup(self): self.cluster = "bravo" self.name = "b010" self.inventory = Inventory() self.inventory.clear() self.inventory.generate() print "GENERATION COMPLETE"
def _connect_to_inventory(self): """connects to the inventory and prints an error if not successfull""" self.inventory_name = "test" try: # TODO: port number is missing # TODO: should be imported from cloudmesh_server.yaml # db = connect(self.inventory_name) self.inventory = Inventory() except: self.Inventory = None raise Exception("ERROR: connection to inventory failed")
def activate_cm_shell_inventory(self): self.register_command_topic('cloud','inventory') # # BUG this needs to be done not in activate # self.inventory_name = "test" # port number is missing # should be imported from cloudmesh_server.yaml db = connect(self.inventory_name) self.inventory = Inventory() pass
class BaremetalProvisinerABC: __metaclass__ = ABCMeta # hosts = ['host1', 'host2'] # image = 'hpc' hosts = [] image = [] def __init__(self): self.inventory = Inventory() @abstractmethod def provision(self, hosts, image): self.hosts = hosts self.image = image pass def set_status(self, host_label, status): print("SIM setting", host_label, status) self.inventory.set_attribute(host_label, "cm_provision_status", status) print("SIM ok setting", host_label, status)
def inventory(): mesh_inventory = Inventory() mesh_inventory.clear() mesh_inventory.generate() mesh_inventory.info()
def __init__(self): self.inventory = Inventory()
def __init__(self): self.config_server = cm_config_server().get( "cloudmesh.server.clusters") self.inventory = Inventory()
class pbs_note_builder: def __init__(self, user, host): self.username = user self.hostname = host self.inventory = Inventory() self.fetch_pbs_nodes_info() # get recent pbsnodes info def fetch_pbs_nodes_info(self): pbs = PBS(self.username, self.hostname) self.pbs_nodes_info = pbs.pbsnodes() #print self.pbs_nodes_info def check_node_validation(self, node): node_id_label = self.inventory.get_host_id_label(node) berror = False if node_id_label is None: berror = True else: (node_id, node_label) = node_id_label if not berror: if node_label not in self.pbs_nodes_info.keys(): berror = True if berror: raise NameError( "pbs_note_builder: '{0}' is NOT a valid or existed node.". format(node)) return node_id_label def get_note(self, node): (node_id, node_label) = self.check_node_validation(node) print "{0}-note: {1}".format(node_id, self.pbs_nodes_info[node_label]["note"]) # node is the server name, e.g., i129, i15 # note is a dict, {"attr1": "value1", "attr2": "value2"} # setNote doesn't check the correctness of the attribute-value pair def set_note(self, node, note): (node_id, node_label) = self.check_node_validation(node) # ["note"] ONLY has two type: dict or string prev_note = self.pbs_nodes_info[node_label]["note"] if type(prev_note) is dict: curr_note = deepcopy(prev_note) else: # assume the default note is for 'service' curr_note = {"service": deepcopy(prev_note)} # now curr_note already is a dict # to keep consistency, the keys in note should be lower map(lambda x: str(x).lower(), note.keys()) curr_note.update(note) # convert the dict to a unique string # e.g., "'other': 'test', 'temperature': '10.2', 'service': 'hpc'" #kvpair_list = ", ".join([": ".join(map(lambda x: "'".join(["", str(x), ""]), [key, prev_note[key]])) for key in sorted(prev_note.keys())]) #snote = "".join(['{', kvpair_list, '}']) #sshnote = '"'.join(["", snote, ""]) # try get the dict string with json dumps sshnote = json.dumps(curr_note) # update the note attribute in memory to real node command = " ".join(["pbsnodes -N", sshnote, node_label]) str_ssh = "@".join([self.username, self.hostname]) log.debug( "pbs_note_builder: command ready to execute is: \n > ssh {0} {1}\n" .format(str_ssh, command)) # This operation NEED authorization ... #ssh(str_ssh, command) # set server's temperature # a shortcut of set_note def set_temperature_note(self, node, temp): self.set_one_note(node, "temperature", temp) # set server's service type # a shortcut of set_note def set_service_note(self, node, service): self.set_one_note(node, "service", service) def set_one_note(self, node, attr, value): self.set_note(node, {attr: value})
class cm_temperature: # instance of cm_config_server config_server = None # instance of Inventory inventory = None # pattern of temperature, such as "30 degrees C" patt = re.compile("([0-9+.-]+?)\s+degrees\s+([A-Z])", re.I) def __init__(self): self.config_server = cm_config_server().get( "cloudmesh.server.clusters") self.inventory = Inventory() # get sensor temperature with ipmitool # param: hostname is a uniform hostname as the following # india cluster: i001, i010, i100 # echo cluster: e001, e010 # delta cluster: d001, d010 # bravo cluster: b001, b010 # # return: a dict with the following style: # { # "name": "Domain A FP Temp", # "address": "44h", # "status": "ok", # "entity": "7.1", # "value": "24" # "unit": "C" # } def get_ipmi_temperature(self, hostname): (hostaddr, data) = self.inventory.ipadr_cluster(hostname, "bmc") clustername = data["cm_cluster"] config = self.config_server[clustername]['bmc'] # username and password to access ipmitool password = config['password'] username = config['user'] command = "ipmitool -I lanplus -H {0} -U {1} -P {2} sdr type temperature".format( hostaddr, username, password) # access ipmitool need a proxy server proxyaddr = config['proxy']['ip'] proxyusername = config['proxy']['user'] log.debug("Get temperature for host '{2}' via proxy server '{0}@{1}'".format( proxyusername, proxyaddr, hostname)) try: result = ssh("{0}@{1}".format(proxyusername, proxyaddr), command) except: result = "" dict_result = None if result == "": log.warning( "Cannot access to host '{0}' OR ipmitool failed on host '{0}'".format(hostname)) else: log.debug( "Temperature data retrieved from host '{0}' successfully.".format(hostname)) dict_result = {} lines = result.split("\n") for line in lines: fields = map(lambda x: x.strip(), line.split("|")) name = fields[0] # test and ignore the blank line in the last output if name == "": continue value = "-1" unit = "C" m = self.patt.match(fields[4]) if m: value = m.group(1) unit = m.group(2) dict_result[name] = {"name": fields[0], "address": fields[1], "status": fields[2], "entity": fields[3], "value": value, "unit": unit } return dict_result # get the max temperature on a server # params: tdict is the result from get_ipmi_temperature # unit is a temperature unit either 'C' or 'F' # return: a dictary def parse_max_temp(self, tdict, unit): unit_upper = unit.upper() max_temp = -1 options = { "C": "convert_temp_C2F", "F": "convert_temp_F2C", } if tdict is not None: for name in tdict: if tdict[name]["status"].lower() == "ok": value = round(float(tdict[name]["value"]), 1) if unit_upper != tdict[name]["unit"].upper(): value = getattr(self, options[unit_upper])(value) if value > max_temp: max_temp = value return {"unit": unit_upper, "value": max_temp} # convert temperature from Fahrenheit (F) to Celsius (C) def convert_temp_C2F(self, cvalue): return round(cvalue * 1.8 + 32, 1) # convert temperature from Celsius (C) to Fahrenheit (F) def convert_temp_F2C(self, fvalue): return round((fvalue - 32) / 1.8, 1)
def __init__(self, valid_time=1800): self.data_valid_time = valid_time self.inventory = Inventory()
class Test_Inventory: def setup(self): self.cluster = "bravo" self.name = "b010" self.inventory = Inventory() self.inventory.clear() self.inventory.generate() print "GENERATION COMPLETE" def tearDown(self): pass def test_bootspec(self): self.inventory.generate_bootspec() specs = self.inventory.find ({'cm_type': "inventory", 'cm_key': 'bootspec', 'cm_kind': 'bootspec'}) print specs.count() for spec in specs: print '#', 70 * "-" print '#', spec['cm_id'] print '#', 70 * "-" pprint (spec) assert True def test_getspec(self): self.inventory.generate_bootspec() spec = self.inventory.get_bootspec('ubuntu-2013-07-b') pprint (spec) assert True def test_clear(self): HEADING() self.inventory.clear() def test_find(self): HEADING() r = self.inventory.find ({}) print r.count() assert r.count > 0 def test_host(self): HEADING() data = self.inventory.host(self.name) pprint(data) def test_list(self): HEADING() data = self.inventory.hostlist(self.cluster) # pprint(data) def test_combine(self): attribute = "cm_temp" value = "32" print "SET ATTRIBUTE" print 70 * '=' data = self.inventory.set_attribute(self.name, attribute, value) print 70 * '=' print data print "GET ATTRIBUTE" data = self.inventory.get_attribute(self.name, attribute) print data data = self.inventory.host(self.name) pprint(data) def test_set(self): HEADING() """ data = self.inventory.find({'cm_id': self.name}) for e in data: pprint (e) """ print 70 * '=' """ print "BEFORE" data = self.inventory.host(self.name) pprint(data) """ attribute = "cm_temp" value = "32" print "SET ATTRIBUTE" print 70 * '=' data = self.inventory.set_attribute(self.name, attribute, value) print 70 * '=' print data print "GET ATTRIBUTE" data = self.inventory.get_attribute(self.name, attribute) print data def test_i066(self): HEADING() name = "i066" attribute = "cm_doesnotexist" print "GET ATTRIBUTE" data = self.inventory.get_attribute(name, attribute) print data """
class Test_Inventory: def setup(self): self.cluster = "bravo" self.name = "b010" self.inventory = Inventory() self.inventory.clear() self.inventory.generate() print("GENERATION COMPLETE") def tearDown(self): pass def test_clear(self): HEADING() self.inventory.clear() def test_find(self): HEADING() r = self.inventory.find({}) print(r.count()) assert r.count > 0 def test_host(self): HEADING() data = self.inventory.host(self.name) pprint(data) def test_list(self): HEADING() data = self.inventory.hostlist(self.cluster) # pprint(data) def test_combine(self): attribute = "cm_temp" value = "32" print("SET ATTRIBUTE") print(70 * '=') data = self.inventory.set_attribute(self.name, attribute, value) print(70 * '=') print(data) print("GET ATTRIBUTE") data = self.inventory.get_attribute(self.name, attribute) print(data) data = self.inventory.host(self.name) pprint(data) def test_set(self): HEADING() """ data = self.inventory.find({'cm_id': self.name}) for e in data: pprint (e) """ print(70 * '=') """ print "BEFORE" data = self.inventory.host(self.name) pprint(data) """ attribute = "cm_temp" value = "32" print("SET ATTRIBUTE") print(70 * '=') data = self.inventory.set_attribute(self.name, attribute, value) print(70 * '=') print(data) print("GET ATTRIBUTE") data = self.inventory.get_attribute(self.name, attribute) print(data) def test_i066(self): HEADING() name = "i066" attribute = "cm_doesnotexist" print("GET ATTRIBUTE") data = self.inventory.get_attribute(name, attribute) print(data) """
def __init__(self, user, host): self.username = user self.hostname = host self.inventory = Inventory() self.fetch_pbs_nodes_info()
from cloudmesh_common.tables import table_printer from cloudmesh_common.util import cond_decorator from cloudmesh.config.cm_config import cm_config_server from flask.ext.principal import Permission, RoleNeed from cloudmesh_base.logger import LOGGER from pprint import pprint log = LOGGER(__file__) admin_permission = Permission(RoleNeed('admin')) import hostlist inventory = Inventory() # inventory.clear() # inventory.generate() @inventory_module.route('/inventory/') @login_required @admin_permission.require(http_exception=403) def display_inventory(): time_now = datetime.now().strftime("%Y-%m-%d %H:%M") # inventory.refresh() clusters = ["bravo", "india", "delta", "echo"] return render_template('mesh/inventory/mesh_inventory.html',
class pbs_note_builder: def __init__(self, user, host): self.username = user self.hostname = host self.inventory = Inventory() self.fetch_pbs_nodes_info() # get recent pbsnodes info def fetch_pbs_nodes_info(self): pbs = PBS(self.username, self.hostname) self.pbs_nodes_info = pbs.pbsnodes() #print self.pbs_nodes_info def check_node_validation(self, node): node_id_label = self.inventory.get_host_id_label(node) berror = False if node_id_label is None: berror = True else: (node_id, node_label) = node_id_label if not berror: if node_label not in self.pbs_nodes_info.keys(): berror = True if berror: raise NameError("pbs_note_builder: '{0}' is NOT a valid or existed node.".format(node)) return node_id_label def get_note(self, node): (node_id, node_label) = self.check_node_validation(node) print "{0}-note: {1}".format(node_id, self.pbs_nodes_info[node_label]["note"]) # node is the server name, e.g., i129, i15 # note is a dict, {"attr1": "value1", "attr2": "value2"} # setNote doesn't check the correctness of the attribute-value pair def set_note(self, node, note): (node_id, node_label) = self.check_node_validation(node) # ["note"] ONLY has two type: dict or string prev_note = self.pbs_nodes_info[node_label]["note"] if type(prev_note) is dict: curr_note = deepcopy(prev_note) else: # assume the default note is for 'service' curr_note = {"service": deepcopy(prev_note)} # now curr_note already is a dict # to keep consistency, the keys in note should be lower map(lambda x: str(x).lower(), note.keys()) curr_note.update(note) # convert the dict to a unique string # e.g., "'other': 'test', 'temperature': '10.2', 'service': 'hpc'" #kvpair_list = ", ".join([": ".join(map(lambda x: "'".join(["", str(x), ""]), [key, prev_note[key]])) for key in sorted(prev_note.keys())]) #snote = "".join(['{', kvpair_list, '}']) #sshnote = '"'.join(["", snote, ""]) # try get the dict string with json dumps sshnote = json.dumps(curr_note) # update the note attribute in memory to real node command = " ".join(["pbsnodes -N", sshnote, node_label]) str_ssh = "@".join([self.username, self.hostname]) log.debug("pbs_note_builder: command ready to execute is: \n > ssh {0} {1}\n".format(str_ssh, command)) # This operation NEED authorization ... #ssh(str_ssh, command) # set server's temperature # a shortcut of set_note def set_temperature_note(self, node, temp): self.set_one_note(node, "temperature", temp) # set server's service type # a shortcut of set_note def set_service_note(self, node, service): self.set_one_note(node, "service", service) def set_one_note(self, node, attr, value): self.set_note(node, {attr: value})
class cm_shell_inventory: """opt_example class""" def info_cm_shell_inventory(self): print "%20s =" % "DBNAME", self.inventory_name def activate_cm_shell_inventory(self): self.register_command_topic('cloud', 'inventory') # # BUG this needs to be done not in activate # self.inventory_name = "test" # port number is missing # should be imported from cloudmesh_server.yaml db = connect(self.inventory_name) self.inventory = Inventory() pass @command def do_inventory(self, args, arguments): """ Usage: inventory clean inventory create image DESCRIPTION inventory create server [dynamic] DESCRIPTION inventory create service [dynamic] DESCRIPTION inventory exists server NAME inventory exists service NAME inventory inventory print inventory info [--cluster=CLUSTER] [--server=SERVER] inventory list [--cluster=CLUSTER] [--server=SERVER] inventory server NAME inventory service NAME Manages the inventory clean cleans the inventory server define servers Arguments: DESCRIPTION The hostlist"i[009-011],i[001-002]" NAME The name of a service or server Options: v verbose mode """ #if arguments["v"]: log.info(arguments) log.info(args) if args == "": log.info("print inventory") if arguments["clean"]: log.info("clean the inventory") return if arguments["print"]: log.info("print the inventory") r = self.inventory.find({}) for e in r: pprint(e) return if arguments["info"]: print print "Inventory Information" print "---------------------" print if not (arguments["--cluster"] or arguments["--server"]): print self.inventory.info() print if arguments["--cluster"] and not arguments["--server"]: name = arguments["--cluster"] r = self.inventory.cluster(name) pprint(r) return for kind in ["server", "service", "image"]: if arguments["info"] and arguments[kind]: name = arguments["NAME"] kind = arguments[kind] self.inventory.print_kind(serv, name) return if arguments["create"] and arguments["server"]: hostlist = arguments["DESCRIPTION"] log.info("create servers" + hostlist) return if arguments["create"] and arguments["service"]: hostlist = arguments["DESCRIPTION"] log.info("create services" + hostlist) return if arguments["exists"] and arguments["server"]: name = arguments["NAME"] if self.inventory.exists("server", name): print "true" else: print "false" return if arguments["exists"] and arguments["service"]: name = arguments["NAME"] if self.inventory.exists("service", name): print "true" else: print "false" return
class RackData: # ONLY for debug test MY_DEBUG_FLAG = False # data valid time, unit second data_valid_time = 0 inventory = None TEMPERATURE_NAME = "temperature" SERVICE_NAME = "service" LOCATION_TEMP = "temp" # three status of server refresh STATUS_NOT_READY = "not_ready" STATUS_REFRESH = "refresh" STATUS_READY = "ready" # default data valid time is set to 1800 seconds def __init__(self, valid_time=1800): self.data_valid_time = valid_time self.inventory = Inventory() def mydebug(self, msg): if self.MY_DEBUG_FLAG: log.debug(msg) # get/set data valid time in mongo db def get_data_valid_time(self): return self.data_valid_time def set_data_valid_time(self, valid_time): self.data_valid_time = valid_time if valid_time > 0 else 0 # change refresh status def set_status_start_refresh(self, service_type, rack_name): self.mydebug( "set status start refresh for {0}-{1}".format(rack_name, service_type)) query_dict = self.get_rack_query_dict( service_type, rack_name, self.LOCATION_TEMP) curr_time = datetime.now() element = { "rack_status": self.STATUS_REFRESH, "updated_node": 0, "cm_refresh": curr_time, } racks = {} for rack_name in query_dict: self.mydebug( "set_status_start_refresh rackname is: {0} ".format(rack_name)) tmp_racks = self.inventory.get_clusters(rack_name) racks[rack_name] = tmp_racks[0] # self.mydebug("set_status_start_refresh racks[{0}] is: {1}".format(rack_name, racks[rack_name])) element['data'] = dict((h, None) for h in racks[rack_name]['cm_value']) self.partly_update(query_dict[rack_name], {"$set": element}) self.mydebug( "Exit from status start refresh for {0}-{1}".format(rack_name, service_type)) # get valid list of rack name # result: if rack_name is 'all', then ['india', 'echo', ...] # else if rack_name is a valid rack name, then ['rack_name.lower()'] # else None def get_rack_name_list(self, rack_name="all"): self.mydebug("enter into get_rack_name_list of {0}".format(rack_name)) racks = self.inventory.get_clusters() rack_name_list = [rack['cm_cluster'] for rack in racks] rack_name_lower = rack_name.lower() if rack_name_lower != 'all': rack_name_list = [ rack_name_lower] if rack_name_lower in rack_name_list else None # self.mydebug("exit from get_rack_name_list {0}".format(rack_name_list)) return rack_name_list # generate a query dict to query the rack table in inventory # location: 'temp' means cm_key is 'rack_temp_*' which is a temporary status table # None means cm_key is 'rack_*', which is a permanent status table # result: if rack_name is 'all', then {"india": {query ...}, "echo": {query ...}, ... } # else if rack_name is valid then {"rack_name": {query ...}} # else None def get_rack_query_dict(self, service_type, rack_name, location=None): rack_name_list = self.get_rack_name_list(rack_name) if rack_name_list is None: return None query_dict = {} for rack_name in rack_name_list: query = { "cm_type": "inventory", "cm_kind": "rack", "cm_id": rack_name } if location: query["cm_key"] = "rack_{1}_{0}".format(service_type, location) else: query["cm_key"] = "rack_{0}".format(service_type) query_dict[rack_name] = query self.mydebug("get_rack_query_dict of {0}".format(query_dict)) return query_dict def get_rack_info(self, query_dict): if query_dict is None: return None rack_info_dict = {} for rack_name in query_dict: rack_info_dict[rack_name] = self.inventory.find_one( query_dict[rack_name]) # self.mydebug("get_rack_info of {0}".format(rack_info_dict.keys())) return rack_info_dict # result: {'rack_name': True, 'rack_name': False, ...} def can_start_refresh(self, service_type, rack_name): self.mydebug( "enter into can_start_refresh og {0}-{1}".format(rack_name, service_type)) query_dict = self.get_rack_query_dict( service_type, rack_name, self.LOCATION_TEMP) rack_info_dict = self.get_rack_info(query_dict) if rack_info_dict is None: return None refresh_dict = {} for rack_name in rack_info_dict: flag_refresh = False rack_info = rack_info_dict[rack_name] status = rack_info['rack_status'] if status == self.STATUS_NOT_READY: flag_refresh = True elif status == self.STATUS_READY or status == self.STATUS_REFRESH: flag_refresh = self.is_db_data_expired(rack_info['cm_refresh']) refresh_dict[rack_name] = flag_refresh self.mydebug("can_start_refresh of {0}".format(refresh_dict)) return refresh_dict # projection query def partly_query(self, query, partly_view): if partly_view: return self.inventory.db_inventory.find_one(query, partly_view) return self.inventory.find_one(query) # update a part of document in database def partly_update(self, query, value, flag_upsert=False, flag_multi=False): if not value: return self.inventory.update(query) return self.inventory.db_inventory.update(query, value, upsert=flag_upsert, multi=flag_multi) # Usually called in a different process # update the temperature of a server # NOT support to update 'all' # The update of 'all' MUST be divided into serveral updates of each independent rack of 'all' # update the information of a server def server_refresh_update_temperature(self, rack_name, server, data): self.mydebug("enter into server_refresh_update_temperature update server {0} for {1}".format( server, rack_name)) service_type = self.TEMPERATURE_NAME flag_refresh_success = False query_dict = self.get_rack_query_dict( service_type, rack_name, self.LOCATION_TEMP) rack_info_dict = self.get_rack_info(query_dict) rack_info = None if rack_info_dict: rack_info = rack_info_dict[rack_name] time_now = datetime.now() element = {} element['cm_refresh'] = time_now element['data.{0}'.format(server)] = data # atomic update update_result = self.inventory.db_inventory.find_and_modify(query=query_dict[rack_name], update={ '$inc': {'updated_node': 1}, '$set': element}, new=True, fields={'updated_node': 1}) if update_result['updated_node'] == rack_info['max_node']: self.partly_update( query_dict[rack_name], {'$set': {'rack_status': self.STATUS_READY}}) flag_refresh_success = True # update the normal record, which provides information to WEB page # After data of all servers in this rack are collected if flag_refresh_success: query_dict = self.get_rack_query_dict(service_type, rack_name) element = {} element['rack_status'] = self.STATUS_READY element['cm_refresh'] = time_now rack_info['data'][server] = data element['data'] = rack_info['data'] self.mydebug( "server_refresh_update_data, update rack {0} with data".format(rack_name)) self.partly_update(query_dict[rack_name], {'$set': element}) return flag_refresh_success # Usually called in a different process # update the service of a server def server_refresh_update_service(self, rack_name, data): self.mydebug( "enter into server_refresh_update_service update rack {0}".format(rack_name)) service_type = self.SERVICE_NAME flag_refresh_success = False query_dict = {} for rack_name in data: tmp_result = self.get_rack_query_dict( service_type, rack_name, self.LOCATION_TEMP) query_dict[rack_name] = tmp_result[rack_name] rack_info_dict = self.get_rack_info(query_dict) if rack_info_dict is None: return flag_refresh_success for rack_name in rack_info_dict: rack_info = rack_info_dict[rack_name] time_now = datetime.now() element = {} element['cm_refresh'] = time_now element['updated_node'] = rack_info['max_node'] element['rack_status'] = self.STATUS_READY element['data'] = data[rack_name] self.partly_update(query_dict[rack_name], {'$set': element}) flag_refresh_success = True # update the normal record, which provides information to WEB page # After data of all servers in this rack are collected if flag_refresh_success: temp_query_dict = self.get_rack_query_dict( service_type, rack_name) element = {} element['rack_status'] = self.STATUS_READY element['cm_refresh'] = time_now element['data'] = data[rack_name] self.mydebug( "server_refresh_update_data, update rack {0} with data".format(rack_name)) self.partly_update( temp_query_dict[rack_name], {'$set': element}) return flag_refresh_success # get data of a rack def get_rack_status_data(self, service_type, rack_name): rack_info_dict = self.get_rack_info( self.get_rack_query_dict(service_type, rack_name)) if rack_info_dict is None: return None dict_result = {} for rack_name in rack_info_dict: dict_result[rack_name] = rack_info_dict[rack_name]["data"] self.mydebug( "exit from get_rack_status_data {0}-{1} with data".format(rack_name, service_type)) return dict_result # get temperature data of a rack def get_rack_temperature_data(self, rack_name): return self.get_rack_status_data(self.TEMPERATURE_NAME, rack_name) # get service data of a rack def get_rack_service_data(self, rack_name): return self.get_rack_status_data(self.SERVICE_NAME, rack_name) # whether the data of a rack is ready or not def is_rack_data_ready(self, service_type, rack_name, refresh_flag=False): location = self.LOCATION_TEMP if refresh_flag else None query_dict = self.get_rack_query_dict( service_type, rack_name, location) rack_info_dict = self.get_rack_info(query_dict) if rack_info_dict is None: return False flag_ready = False for rack_name in rack_info_dict: #self.mydebug("{0} data is {1}".format(rack_name, rack_info_dict[rack_name])) rack_info = rack_info_dict[rack_name] if rack_info['rack_status'] == self.STATUS_READY: flag_ready = not self.is_db_data_expired( rack_info["cm_refresh"]) if not flag_ready: break return flag_ready # check data is ready or not in temp refresh db def is_refresh_rack_data_ready(self, service_type, rack_name): return self.is_rack_data_ready(service_type, rack_name, True) # whether the temperature data of a rack is ready or not def is_rack_temperature_ready(self, rack_name): return self.is_rack_data_ready(self.TEMPERATURE_NAME) # whether the service data of a rack is ready or not def is_rack_service_ready(self, rack_name): return self.is_rack_data_ready(self.SERVICE_NAME, rack_name) # check data in mongo db expired or not def is_db_data_expired(self, db_time): flag_expired = False time_interval = timedelta(seconds=self.data_valid_time) if (datetime.now() > db_time + time_interval): flag_expired = True return flag_expired
class cm_shell_inventory: """opt_example class""" def info_cm_shell_inventory(self): print "%20s =" % "DBNAME", self.inventory_name def activate_cm_shell_inventory(self): self.register_command_topic('cloud','inventory') # # BUG this needs to be done not in activate # self.inventory_name = "test" # port number is missing # should be imported from cloudmesh_server.yaml db = connect(self.inventory_name) self.inventory = Inventory() pass @command def do_inventory(self, args, arguments): """ Usage: inventory clean inventory create image DESCRIPTION inventory create server [dynamic] DESCRIPTION inventory create service [dynamic] DESCRIPTION inventory exists server NAME inventory exists service NAME inventory inventory print inventory info [--cluster=CLUSTER] [--server=SERVER] inventory list [--cluster=CLUSTER] [--server=SERVER] inventory server NAME inventory service NAME Manages the inventory clean cleans the inventory server define servers Arguments: DESCRIPTION The hostlist"i[009-011],i[001-002]" NAME The name of a service or server Options: v verbose mode """ #if arguments["v"]: log.info(arguments) log.info(args) if args == "": log.info("print inventory") if arguments["clean"]: log.info("clean the inventory") return if arguments["print"]: log.info("print the inventory") r = self.inventory.find({}) for e in r: pprint(e) return if arguments["info"]: print print "Inventory Information" print "---------------------" print if not (arguments["--cluster"] or arguments["--server"]): print self.inventory.info() print if arguments["--cluster"] and not arguments["--server"]: name = arguments["--cluster"] r = self.inventory.cluster(name) pprint(r) return for kind in ["server", "service", "image"]: if arguments["info"] and arguments[kind]: name = arguments["NAME"] kind = arguments[kind] self.inventory.print_kind(serv, name) return if arguments["create"] and arguments["server"]: hostlist = arguments["DESCRIPTION"] log.info("create servers" + hostlist) return if arguments["create"] and arguments["service"]: hostlist = arguments["DESCRIPTION"] log.info("create services" + hostlist) return if arguments["exists"] and arguments["server"]: name = arguments["NAME"] if self.inventory.exists("server", name): print "true" else: print "false" return if arguments["exists"] and arguments["service"]: name = arguments["NAME"] if self.inventory.exists("service", name): print "true" else: print "false" return
class RackData: # ONLY for debug test MY_DEBUG_FLAG = False # data valid time, unit second data_valid_time = 0 inventory = None TEMPERATURE_NAME = "temperature" SERVICE_NAME = "service" LOCATION_TEMP = "temp" # three status of server refresh STATUS_NOT_READY = "not_ready" STATUS_REFRESH = "refresh" STATUS_READY = "ready" # default data valid time is set to 1800 seconds def __init__(self, valid_time=1800): self.data_valid_time = valid_time self.inventory = Inventory() def mydebug(self, msg): if self.MY_DEBUG_FLAG: log.debug(msg) # get/set data valid time in mongo db def get_data_valid_time(self): return self.data_valid_time def set_data_valid_time(self, valid_time): self.data_valid_time = valid_time if valid_time > 0 else 0 # change refresh status def set_status_start_refresh(self, service_type, rack_name): self.mydebug("set status start refresh for {0}-{1}".format( rack_name, service_type)) query_dict = self.get_rack_query_dict(service_type, rack_name, self.LOCATION_TEMP) curr_time = datetime.now() element = { "rack_status": self.STATUS_REFRESH, "updated_node": 0, "cm_refresh": curr_time, } racks = {} for rack_name in query_dict: self.mydebug( "set_status_start_refresh rackname is: {0} ".format(rack_name)) tmp_racks = self.inventory.get_clusters(rack_name) racks[rack_name] = tmp_racks[0] #self.mydebug("set_status_start_refresh racks[{0}] is: {1}".format(rack_name, racks[rack_name])) element['data'] = dict( (h, None) for h in racks[rack_name]['cm_value']) self.partly_update(query_dict[rack_name], {"$set": element}) self.mydebug("Exit from status start refresh for {0}-{1}".format( rack_name, service_type)) # get valid list of rack name # result: if rack_name is 'all', then ['india', 'echo', ...] # else if rack_name is a valid rack name, then ['rack_name.lower()'] # else None def get_rack_name_list(self, rack_name="all"): self.mydebug("enter into get_rack_name_list of {0}".format(rack_name)) racks = self.inventory.get_clusters() rack_name_list = [rack['cm_cluster'] for rack in racks] rack_name_lower = rack_name.lower() if rack_name_lower != 'all': rack_name_list = [rack_name_lower ] if rack_name_lower in rack_name_list else None #self.mydebug("exit from get_rack_name_list {0}".format(rack_name_list)) return rack_name_list # generate a query dict to query the rack table in inventory # location: 'temp' means cm_key is 'rack_temp_*' which is a temporary status table # None means cm_key is 'rack_*', which is a permanent status table # result: if rack_name is 'all', then {"india": {query ...}, "echo": {query ...}, ... } # else if rack_name is valid then {"rack_name": {query ...}} # else None def get_rack_query_dict(self, service_type, rack_name, location=None): rack_name_list = self.get_rack_name_list(rack_name) if rack_name_list is None: return None query_dict = {} for rack_name in rack_name_list: query = { "cm_type": "inventory", "cm_kind": "rack", "cm_id": rack_name } if location: query["cm_key"] = "rack_{1}_{0}".format(service_type, location) else: query["cm_key"] = "rack_{0}".format(service_type) query_dict[rack_name] = query self.mydebug("get_rack_query_dict of {0}".format(query_dict)) return query_dict def get_rack_info(self, query_dict): if query_dict is None: return None rack_info_dict = {} for rack_name in query_dict: rack_info_dict[rack_name] = self.inventory.find_one( query_dict[rack_name]) #self.mydebug("get_rack_info of {0}".format(rack_info_dict.keys())) return rack_info_dict # result: {'rack_name': True, 'rack_name': False, ...} def can_start_refresh(self, service_type, rack_name): self.mydebug("enter into can_start_refresh og {0}-{1}".format( rack_name, service_type)) query_dict = self.get_rack_query_dict(service_type, rack_name, self.LOCATION_TEMP) rack_info_dict = self.get_rack_info(query_dict) if rack_info_dict is None: return None refresh_dict = {} for rack_name in rack_info_dict: flag_refresh = False rack_info = rack_info_dict[rack_name] status = rack_info['rack_status'] if status == self.STATUS_NOT_READY: flag_refresh = True elif status == self.STATUS_READY or status == self.STATUS_REFRESH: flag_refresh = self.is_db_data_expired(rack_info['cm_refresh']) refresh_dict[rack_name] = flag_refresh self.mydebug("can_start_refresh of {0}".format(refresh_dict)) return refresh_dict # projection query def partly_query(self, query, partly_view): if partly_view: return self.inventory.db_inventory.find_one(query, partly_view) return self.inventory.find_one(query) # update a part of document in database def partly_update(self, query, value, flag_upsert=False, flag_multi=False): if not value: return self.inventory.update(query) return self.inventory.db_inventory.update(query, value, upsert=flag_upsert, multi=flag_multi) # Usually called in a different process # update the temperature of a server # NOT support to update 'all' # The update of 'all' MUST be divided into serveral updates of each independent rack of 'all' # update the information of a server def server_refresh_update_temperature(self, rack_name, server, data): self.mydebug( "enter into server_refresh_update_temperature update server {0} for {1}" .format(server, rack_name)) service_type = self.TEMPERATURE_NAME flag_refresh_success = False query_dict = self.get_rack_query_dict(service_type, rack_name, self.LOCATION_TEMP) rack_info_dict = self.get_rack_info(query_dict) rack_info = None if rack_info_dict: rack_info = rack_info_dict[rack_name] time_now = datetime.now() element = {} element['cm_refresh'] = time_now element['data.{0}'.format(server)] = data # atomic update update_result = self.inventory.db_inventory.find_and_modify( query=query_dict[rack_name], update={ '$inc': { 'updated_node': 1 }, '$set': element }, new=True, fields={'updated_node': 1}) if update_result['updated_node'] == rack_info['max_node']: self.partly_update( query_dict[rack_name], {'$set': { 'rack_status': self.STATUS_READY }}) flag_refresh_success = True # update the normal record, which provides information to WEB page # After data of all servers in this rack are collected if flag_refresh_success: query_dict = self.get_rack_query_dict(service_type, rack_name) element = {} element['rack_status'] = self.STATUS_READY element['cm_refresh'] = time_now rack_info['data'][server] = data element['data'] = rack_info['data'] self.mydebug( "server_refresh_update_data, update rack {0} with data". format(rack_name)) self.partly_update(query_dict[rack_name], {'$set': element}) return flag_refresh_success # Usually called in a different process # update the service of a server def server_refresh_update_service(self, rack_name, data): self.mydebug( "enter into server_refresh_update_service update rack {0}".format( rack_name)) service_type = self.SERVICE_NAME flag_refresh_success = False query_dict = {} for rack_name in data: tmp_result = self.get_rack_query_dict(service_type, rack_name, self.LOCATION_TEMP) query_dict[rack_name] = tmp_result[rack_name] rack_info_dict = self.get_rack_info(query_dict) if rack_info_dict is None: return flag_refresh_success for rack_name in rack_info_dict: rack_info = rack_info_dict[rack_name] time_now = datetime.now() element = {} element['cm_refresh'] = time_now element['updated_node'] = rack_info['max_node'] element['rack_status'] = self.STATUS_READY element['data'] = data[rack_name] self.partly_update(query_dict[rack_name], {'$set': element}) flag_refresh_success = True # update the normal record, which provides information to WEB page # After data of all servers in this rack are collected if flag_refresh_success: temp_query_dict = self.get_rack_query_dict( service_type, rack_name) element = {} element['rack_status'] = self.STATUS_READY element['cm_refresh'] = time_now element['data'] = data[rack_name] self.mydebug( "server_refresh_update_data, update rack {0} with data". format(rack_name)) self.partly_update(temp_query_dict[rack_name], {'$set': element}) return flag_refresh_success # get data of a rack def get_rack_status_data(self, service_type, rack_name): rack_info_dict = self.get_rack_info( self.get_rack_query_dict(service_type, rack_name)) if rack_info_dict is None: return None dict_result = {} for rack_name in rack_info_dict: dict_result[rack_name] = rack_info_dict[rack_name]["data"] self.mydebug("exit from get_rack_status_data {0}-{1} with data".format( rack_name, service_type)) return dict_result # get temperature data of a rack def get_rack_temperature_data(self, rack_name): return self.get_rack_status_data(self.TEMPERATURE_NAME, rack_name) # get service data of a rack def get_rack_service_data(self, rack_name): return self.get_rack_status_data(self.SERVICE_NAME, rack_name) # whether the data of a rack is ready or not def is_rack_data_ready(self, service_type, rack_name, refresh_flag=False): location = self.LOCATION_TEMP if refresh_flag else None query_dict = self.get_rack_query_dict(service_type, rack_name, location) rack_info_dict = self.get_rack_info(query_dict) if rack_info_dict is None: return False flag_ready = False for rack_name in rack_info_dict: #self.mydebug("{0} data is {1}".format(rack_name, rack_info_dict[rack_name])) rack_info = rack_info_dict[rack_name] if rack_info['rack_status'] == self.STATUS_READY: flag_ready = not self.is_db_data_expired( rack_info["cm_refresh"]) if not flag_ready: break return flag_ready # check data is ready or not in temp refresh db def is_refresh_rack_data_ready(self, service_type, rack_name): return self.is_rack_data_ready(service_type, rack_name, True) # whether the temperature data of a rack is ready or not def is_rack_temperature_ready(self, rack_name): return self.is_rack_data_ready(self.TEMPERATURE_NAME) # whether the service data of a rack is ready or not def is_rack_service_ready(self, rack_name): return self.is_rack_data_ready(self.SERVICE_NAME, rack_name) # check data in mongo db expired or not def is_db_data_expired(self, db_time): flag_expired = False time_interval = timedelta(seconds=self.data_valid_time) if (datetime.now() > db_time + time_interval): flag_expired = True return flag_expired
class Test_Inventory: def setup(self): self.cluster = "bravo" self.name = "b010" self.inventory = Inventory() self.inventory.clear() self.inventory.generate() print "GENERATION COMPLETE" def tearDown(self): pass def test_bootspec(self): self.inventory.generate_bootspec() specs = self.inventory.find({ 'cm_type': "inventory", 'cm_key': 'bootspec', 'cm_kind': 'bootspec' }) print specs.count() for spec in specs: print '#', 70 * "-" print '#', spec['cm_id'] print '#', 70 * "-" pprint(spec) assert True def test_getspec(self): self.inventory.generate_bootspec() spec = self.inventory.get_bootspec('ubuntu-2013-07-b') pprint(spec) assert True def test_clear(self): HEADING() self.inventory.clear() def test_find(self): HEADING() r = self.inventory.find({}) print r.count() assert r.count > 0 def test_host(self): HEADING() data = self.inventory.host(self.name) pprint(data) def test_list(self): HEADING() data = self.inventory.hostlist(self.cluster) # pprint(data) def test_combine(self): attribute = "cm_temp" value = "32" print "SET ATTRIBUTE" print 70 * '=' data = self.inventory.set_attribute(self.name, attribute, value) print 70 * '=' print data print "GET ATTRIBUTE" data = self.inventory.get_attribute(self.name, attribute) print data data = self.inventory.host(self.name) pprint(data) def test_set(self): HEADING() """ data = self.inventory.find({'cm_id': self.name}) for e in data: pprint (e) """ print 70 * '=' """ print "BEFORE" data = self.inventory.host(self.name) pprint(data) """ attribute = "cm_temp" value = "32" print "SET ATTRIBUTE" print 70 * '=' data = self.inventory.set_attribute(self.name, attribute, value) print 70 * '=' print data print "GET ATTRIBUTE" data = self.inventory.get_attribute(self.name, attribute) print data def test_i066(self): HEADING() name = "i066" attribute = "cm_doesnotexist" print "GET ATTRIBUTE" data = self.inventory.get_attribute(name, attribute) print data """
class cm_temperature: # instance of cm_config_server config_server = None # instance of Inventory inventory = None # pattern of temperature, such as "30 degrees C" patt = re.compile("([0-9+.-]+?)\s+degrees\s+([A-Z])", re.I) def __init__(self): self.config_server = cm_config_server().get( "cloudmesh.server.clusters") self.inventory = Inventory() # get sensor temperature with ipmitool # param: hostname is a uniform hostname as the following # india cluster: i001, i010, i100 # echo cluster: e001, e010 # delta cluster: d001, d010 # bravo cluster: b001, b010 # # return: a dict with the following style: # { # "name": "Domain A FP Temp", # "address": "44h", # "status": "ok", # "entity": "7.1", # "value": "24" # "unit": "C" # } def get_ipmi_temperature(self, hostname): (hostaddr, data) = self.inventory.ipadr_cluster(hostname, "bmc") clustername = data["cm_cluster"] config = self.config_server[clustername]['bmc'] # username and password to access ipmitool password = config['password'] username = config['user'] command = "ipmitool -I lanplus -H {0} -U {1} -P {2} sdr type temperature".format( hostaddr, username, password) # access ipmitool need a proxy server proxyaddr = config['proxy']['ip'] proxyusername = config['proxy']['user'] log.debug( "Get temperature for host '{2}' via proxy server '{0}@{1}'".format( proxyusername, proxyaddr, hostname)) try: result = ssh("{0}@{1}".format(proxyusername, proxyaddr), command) except: result = "" dict_result = None if result == "": log.warning( "Cannot access to host '{0}' OR ipmitool failed on host '{0}'". format(hostname)) else: log.debug( "Temperature data retrieved from host '{0}' successfully.". format(hostname)) dict_result = {} lines = result.split("\n") for line in lines: fields = map(lambda x: x.strip(), line.split("|")) name = fields[0] # test and ignore the blank line in the last output if name == "": continue value = "-1" unit = "C" m = self.patt.match(fields[4]) if m: value = m.group(1) unit = m.group(2) dict_result[name] = { "name": fields[0], "address": fields[1], "status": fields[2], "entity": fields[3], "value": value, "unit": unit } return dict_result # get the max temperature on a server # params: tdict is the result from get_ipmi_temperature # unit is a temperature unit either 'C' or 'F' # return: a dictary def parse_max_temp(self, tdict, unit): unit_upper = unit.upper() max_temp = -1 options = { "C": "convert_temp_C2F", "F": "convert_temp_F2C", } if tdict is not None: for name in tdict: if tdict[name]["status"].lower() == "ok": value = round(float(tdict[name]["value"]), 1) if unit_upper != tdict[name]["unit"].upper(): value = getattr(self, options[unit_upper])(value) if value > max_temp: max_temp = value return {"unit": unit_upper, "value": max_temp} # convert temperature from Fahrenheit (F) to Celsius (C) def convert_temp_C2F(self, cvalue): return round(cvalue * 1.8 + 32, 1) # convert temperature from Celsius (C) to Fahrenheit (F) def convert_temp_F2C(self, fvalue): return round((fvalue - 32) / 1.8, 1)