コード例 #1
0
ファイル: tasks.py プロジェクト: rowlandifeanyi17/cloudmesh
def pbs_service(rack_name):
    '''
    get all server type with the help of PBS
    '''
    log.debug("pbs_service_task recieved")
    rack_work = RackWork()
    tdict = rack_work.pbs_service(rack_name)
    rackdata = RackData()
    rackdata.server_refresh_update_service(rack_name, tdict)
コード例 #2
0
ファイル: tasks.py プロジェクト: rowlandifeanyi17/cloudmesh
def temperature(host_name, rack_name, unit):
    '''
    get the temperature of 'host_name' with the help of ipmi API
    '''
    log.debug("temperature_task recieved")
    ipmi_temp = Temperature()
    tdict = ipmi_temp.get_ipmi_temperature(host_name)
    rackdata = RackData()
    rackdata.server_refresh_update_temperature(rack_name, host_name, tdict)
コード例 #3
0
ファイル: glow_rest.py プロジェクト: cloudmesh/glow
def read_data_from_mongodb(rack_name="all", unit='C'):
    """ Read data from mongo db """
    rack_data = RackData()
    rack_temperature = RackTemperature()
    raw_data_dict = rack_data.get_rack_temperature_data(rack_name)
    if raw_data_dict is None:
        return None
    
    dict_data = {}
    for rack_name in raw_data_dict:
        dict_data[rack_name] = {}
        for host in raw_data_dict[rack_name]:
            result = rack_temperature.parse_max_temp(raw_data_dict[rack_name][host], unit)
            dict_data[rack_name][host] = result["value"]
    return dict_data
コード例 #4
0
 def __init__(self, username=None, valid_time=1800):
     self.username = username
     self.set_data_valid_time(valid_time)
     self.rackdata = RackData(self.data_valid_time)
     self.rackwork = RackWork(self.username)
コード例 #5
0
class FetchClusterInfo:

    map_progress = None
    username = None
    data_valid_time = 0

    # default data valid time is set to 1800 seconds
    def __init__(self, username=None, valid_time=1800):
        self.username = username
        self.set_data_valid_time(valid_time)
        self.rackdata = RackData(self.data_valid_time)
        self.rackwork = RackWork(self.username)

    # get/set data valid time in mongo db
    def get_data_valid_time(self):
        return self.data_valid_time

    def set_data_valid_time(self, valid_time):
        self.data_valid_time = valid_time if valid_time > 0 else 0

    # start async call to refresh racks
    def start_async_refresh(self, type, rack_name, server):
        if type == self.rackdata.TEMPERATURE_NAME:
            self.rackdata.mydebug(
                "start_async_refresh, BEGIN delay start server {2} of {1}-{0}".
                format(type, rack_name, server))
            temperature.apply_async((server, rack_name, 'C'), queue='rack')
            self.rackdata.mydebug(
                "start_async_refresh, END delay start server {2} of {1}-{0}".
                format(type, rack_name, server))
        elif type == self.rackdata.SERVICE_NAME:
            self.rackdata.mydebug(
                "start_async_refresh, BEGIN delay start pbs_service of {1}-{0}"
                .format(type, rack_name))
            pbs_service.apply_async([rack_name], queue='rack')
            self.rackdata.mydebug(
                "start_async_refresh, END delay start pbs_service of {1}-{0}".
                format(type, rack_name))
        else:
            log.debug("NOT Supported to refresh {0} status of {1}".format(
                type, rack_name))

    # refresh rack status data
    def refresh_rack_data(self, type, rack_name):
        self.rackdata.mydebug("enter into refresh_rack_data of {0}-{1}".format(
            rack_name, type))
        # check the status of 'rack_status'
        refresh_dict = self.rackdata.can_start_refresh(type, rack_name)
        self.map_progress.set_check_refresh_condition()
        if refresh_dict is None:
            return False

        total_hosts = {}
        if type == self.rackdata.TEMPERATURE_NAME:
            for rack_name in refresh_dict:
                if refresh_dict[rack_name]:
                    total_hosts[rack_name] = {
                        "total": 0,
                        "updated": 0,
                        "ratio": 0
                    }
                    # change the status of 'rack_status' to 'refresh'
                    self.rackdata.set_status_start_refresh(type, rack_name)
                    hosts = self.rackdata.inventory.hostlist(rack_name)
                    total_hosts[rack_name]["total"] = len(hosts)
                    self.map_progress.update_data(
                        "temperature_data",
                        {rack_name: total_hosts[rack_name]})
                    # fetch data for each host
                    for server in hosts:
                        self.start_async_refresh(type, rack_name, server)
        elif type == self.rackdata.SERVICE_NAME:
            if reduce(lambda x, y: x or y,
                      [refresh_dict[name] for name in refresh_dict], False):
                for rack_name in refresh_dict:
                    total_hosts[rack_name] = {
                        "total": 0,
                        "updated": 0,
                        "ratio": 0
                    }
                    # change the status of 'rack_status' to 'refresh'
                    self.rackdata.set_status_start_refresh(type, rack_name)
                    hosts = self.rackdata.inventory.hostlist(rack_name)
                    total_hosts[rack_name]["total"] = len(hosts)
                self.map_progress.update_data("service_data", total_hosts)
                # service refresh operation can update all rack ONLY once
                self.start_async_refresh(type, rack_name, None)

        self.rackdata.mydebug(
            "exit from refresh_rack_data of {0}-{1} with refresh data {2}".
            format(rack_name, type, refresh_dict))

        return True

    # refresh rack temperature
    def refresh_rack_temperature(self, rack_name):
        return self.refresh_rack_data(self.rackdata.TEMPERATURE_NAME,
                                      rack_name)

    # refresh rack service
    def refresh_rack_service(self, rack_name):
        return self.refresh_rack_data(self.rackdata.SERVICE_NAME, rack_name)

    # API of generate map
    def start_gen_map(self, service, rack_name):
        t = threading.Thread(target=self.gen_map_thread,
                             args=[service, rack_name])
        t.start()
        return True

    # API of refresh map
    # return True means start refresh process
    #        False means NOT start refresh process, "fresh" gives the reason
    #               if "fresh" is True, that means the data in db is fresh, does NOT need a refresh
    #                             Flase, means ERROR occured in db
    def start_refresh_map(self, service, rack_name):
        result = False
        flag_fresh = False
        refresh_dict = self.rackdata.can_start_refresh(service, rack_name)
        if refresh_dict:
            result = reduce(lambda x, y: x or y,
                            [refresh_dict[name] for name in refresh_dict],
                            False)
            if result:
                t = threading.Thread(target=self.refresh_map_thread,
                                     args=[service, rack_name])
                t.start()
            else:
                flag_fresh = True

        return {"result": result, "fresh": flag_fresh}

    # refresh map thread
    def refresh_map_thread(self, service, rack_name):
        self.get_map_progress(service)
        #self.map_progress.set_refresh_map()
        flag_data_ready = False
        if self.refresh_rack_data(service, rack_name):
            flag_data_ready = self.check_rack_refresh_status(
                service, rack_name)

        if flag_data_ready:
            self.rackwork.generate_map(service, rack_name, True)
        else:
            self.map_progress.set_error_status(
            )  # refresh error, return to user

    # generate map thread
    def gen_map_thread(self, service, rack_name):
        self.get_map_progress(service)

        flag_read_refresh_data = False
        flag_data_ready = False

        if self.rackdata.is_rack_data_ready(service, rack_name):
            flag_data_ready = True
        else:
            self.map_progress.set_load_refresh_map()
            if self.refresh_rack_data(service, rack_name):
                flag_read_refresh_data = True
                flag_data_ready = self.check_rack_refresh_status(
                    service, rack_name)

        if flag_data_ready:
            self.rackwork.generate_map(service, rack_name,
                                       flag_read_refresh_data)
        else:
            self.map_progress.set_error_status(
            )  # read db error, return to user

    # update the status of async  data refresh
    def update_data_refresh_status(self, service):
        data_dict = self.map_progress.get_data("{0}_data".format(service))
        total_count = 0
        total_updated = 0
        for rack_name in data_dict:
            if data_dict[rack_name]["updated"] < data_dict[rack_name]["total"]:
                # query db to get recent update status
                query_dict = self.rackdata.get_rack_query_dict(
                    service, rack_name, self.rackdata.LOCATION_TEMP)
                result_dict = self.rackdata.partly_query(
                    query_dict[rack_name], {"updated_node": 1})
                data_dict[rack_name]["updated"] = result_dict["updated_node"]
            total_count += data_dict[rack_name]["total"]
            total_updated += data_dict[rack_name]["updated"]
        ratio = total_updated * 1.0 / total_count
        self.map_progress.set_async_refresh(round(ratio, 2))

    # check mongo db refresh status
    # default interval time is 200ms, and the max check count is 10000
    def check_rack_refresh_status(self,
                                  service,
                                  rack_name,
                                  interval=0.2,
                                  max_count=10000):
        result = False
        curr_count = 1
        while True:
            sleep(interval)  # default is 200 ms
            # check and update async data refresh status
            self.update_data_refresh_status(service)
            if self.rackdata.is_refresh_rack_data_ready(service, rack_name):
                self.map_progress.set_async_refresh()
                result = True
                break
            curr_count += 1
            if curr_count > max_count:
                break
        return result

    def get_map_progress(self, service):
        if service == self.rackdata.TEMPERATURE_NAME:
            self.map_progress = get_temperature_progress(self.username)
        elif service == self.rackdata.SERVICE_NAME:
            self.map_progress = get_service_progress(self.username)

        return self.map_progress
コード例 #6
0
ファイル: rack_work.py プロジェクト: rajpushkar83/cloudmesh
 def __init__(self, username=None):
     self.username = username
     self.temperature_ipmi = cm_temperature()
     self.rackdata = RackData(self.username)
コード例 #7
0
ファイル: rack_work.py プロジェクト: rajpushkar83/cloudmesh
class RackWork:
    map_progress = None
    username = None

    def __init__(self, username=None):
        self.username = username
        self.temperature_ipmi = cm_temperature()
        self.rackdata = RackData(self.username)

    # thread
    def generate_map(self, service, rack_name, refresh_flag=False):
        # the following begin to generate map
        # class name means the specific class to generate map for different service type
        # method name means the specific method to fetch real data of different service type,
        #     the methods are defined in class FetchClusterInfo
        service_options = {
            "temperature": {
                "class": HeatClusterMap,
                "method": "read_temperature_mongo",
            },
            "service": {
                "class": ServiceClusterMap,
                "method": "read_service_mongo",
            },
        }
        # update progress satus
        self.get_map_progress(service)

        # get location of configuration file, input diag, output image
        dir_base = config_file("")

        server_config = cm_config_server()
        relative_dir_diag = server_config.get("cloudmesh.server.rack.input")
        relative_dir_image = server_config.get(
            "cloudmesh.server.rack.diagrams.{0}".format(service))
        # log.debug("relative dir image, {0}".format(relative_dir_image))
        flask_dir = "static"
        # guess absolute path of cloudmesh_web
        rack_py_dir = pwd().strip().split("/")
        cloudmesh_web_dir = rack_py_dir
        # log.debug("cloudmesh_web dir, {0}".format(cloudmesh_web_dir))
        list_image_dir = [flask_dir] + relative_dir_image.strip().split("/")
        abs_dir_image = "/".join(cloudmesh_web_dir + list_image_dir)
        abs_dir_diag = dir_base + "/" + relative_dir_diag
        # dynamic generate image
        map_class = service_options[service]["class"](
            self.username, rack_name, dir_base, abs_dir_diag, abs_dir_image)
        # get cluster server data
        dict_data = None
        if False:
            dict_data = map_class.genRandomValues()
        else:
            # flag_filter = None if rack_name == "all" else rack_name
            # If user want to customize the action, user can set optional param here
            # by calling map_class.set_optional_param(value)
            # optional param
            aparam = map_class.get_optional_param()
            dict_data = getattr(self, service_options[service]["method"])(
                rack_name, aparam, refresh_flag)

        # update data
        map_class.update(dict_data)
        # plot map
        map_class.plot()

        # get image names
        filename_image = map_class.getImageFilename()
        filename_legend = map_class.getLegendFilename()
        image_size = map_class.getImageSize()
        legend_size = map_class.getImageLegendSize()
        # log.debug("legend size is: {0}".format(legend_size))
        abs_web_path_image = "/".join([""] + list_image_dir + [filename_image])
        abs_web_path_legend = "/".join([""] +
                                       list_image_dir + [filename_legend])
        img_flag = "?" + str(time.time())

        map_data = {
            "map_width": image_size["width"],
            "map_height": image_size["height"],
            "legend_width": legend_size["width"],
            "legend_height": legend_size["height"],
            "map_url": abs_web_path_image + img_flag,
            "legend_url": abs_web_path_legend + img_flag,
        }
        self.map_progress.update_data("map_data", map_data)

    # celery task
    def pbs_service(self, rack_name=None):
        config = cm_config()
        username = config.get("cloudmesh.hpc.username")
        pbs = PBS(username, "india.futuregrid.org")
        dict_pbs_info = pbs.pbsnodes()
        dict_data = {}
        inventory = Inventory()
        racks = inventory.get_clusters()
        for rack in racks:
            rack_name = rack["cm_cluster"]
            dict_data[rack_name] = {}
            hosts = rack["cm_value"]
            for host in hosts:
                (hid, hlabel) = inventory.get_host_id_label(host, "public")
                utype = "unknown"
                if hlabel in dict_pbs_info:
                    server = dict_pbs_info[hlabel]
                    if "note" in server.keys():
                        note_value = server["note"]
                        # to compatible with the future change
                        if type(note_value) is dict:
                            utype = note_value["service"]
                        else:   # currently is a literal string for note
                            utype = note_value
                dict_data[rack_name][hid] = utype
        return dict_data

    # fetch cluster temperature from mongo db
    # params:
    # flag_filter, None or one item in list ['india', 'bravo', 'echo',
    # 'delta']
    def read_temperature_mongo(self, rack_name=None, unit='C', refresh_flag=False):
        # read data from mongo db
        rack_data_dict = self.rackdata.get_rack_temperature_data(rack_name)

        if refresh_flag:
            self.map_progress.set_read_refresh_data(0.7)
        else:
            self.map_progress.set_read_data_from_db(0.7)

        dict_data = {}
        for rack_name in rack_data_dict:
            for host in rack_data_dict[rack_name]:
                result = self.temperature_ipmi.parse_max_temp(
                    rack_data_dict[rack_name][host], unit)
                dict_data[host] = result["value"]

        if refresh_flag:
            self.map_progress.set_read_refresh_data()
        else:
            self.map_progress.set_read_data_from_db()

        return dict_data

    # fetch cluster service from mongo db
    # params:
    # flag_filter, None or one item in list ['india', 'bravo', 'echo',
    # 'delta']
    def read_service_mongo(self, rack_name=None, unit=None, refresh_flag=False):
        # read data from mongo db
        rack_data_dict = self.rackdata.get_rack_service_data(rack_name)

        if refresh_flag:
            self.map_progress.set_read_refresh_data(0.7)
        else:
            self.map_progress.set_read_data_from_db(0.7)

        dict_data = {}
        for rack_name in rack_data_dict:
            for host in rack_data_dict[rack_name]:
                dict_data[host] = rack_data_dict[rack_name][host]

        if refresh_flag:
            self.map_progress.set_read_refresh_data()
        else:
            self.map_progress.set_read_data_from_db()

        return dict_data

    def get_map_progress(self, service):
        if service == self.rackdata.TEMPERATURE_NAME:
            self.map_progress = get_temperature_progress(self.username)
        elif service == self.rackdata.SERVICE_NAME:
            self.map_progress = get_service_progress(self.username)
        return self.map_progress
コード例 #8
0
 def __init__(self, username=None, valid_time=1800):
     self.username = username
     self.set_data_valid_time(valid_time)
     self.rackdata = RackData(self.data_valid_time)
     self.rackwork = RackWork(self.username)
コード例 #9
0
class FetchClusterInfo:

    map_progress = None
    username = None
    data_valid_time = 0
    
    # default data valid time is set to 1800 seconds
    def __init__(self, username=None, valid_time=1800):
        self.username = username
        self.set_data_valid_time(valid_time)
        self.rackdata = RackData(self.data_valid_time)
        self.rackwork = RackWork(self.username)
        
    
    # get/set data valid time in mongo db
    def get_data_valid_time(self):
        return self.data_valid_time
    
    def set_data_valid_time(self, valid_time):
        self.data_valid_time = valid_time if valid_time > 0 else 0
    
    # start async call to refresh racks
    def start_async_refresh(self, type, rack_name, server):
        if type == self.rackdata.TEMPERATURE_NAME:
            self.rackdata.mydebug("start_async_refresh, BEGIN delay start server {2} of {1}-{0}".format(type, rack_name, server))
            temperature.apply_async((server, rack_name, 'C'), queue='rack')
            self.rackdata.mydebug("start_async_refresh, END delay start server {2} of {1}-{0}".format(type, rack_name, server))
        elif type == self.rackdata.SERVICE_NAME:
            self.rackdata.mydebug("start_async_refresh, BEGIN delay start pbs_service of {1}-{0}".format(type, rack_name))
            pbs_service.apply_async([rack_name], queue='rack')
            self.rackdata.mydebug("start_async_refresh, END delay start pbs_service of {1}-{0}".format(type, rack_name))
        else:
            log.debug("NOT Supported to refresh {0} status of {1}".format(type, rack_name))
    
    
    # refresh rack status data
    def refresh_rack_data(self, type, rack_name):
        self.rackdata.mydebug("enter into refresh_rack_data of {0}-{1}".format(rack_name, type))
        # check the status of 'rack_status'
        refresh_dict = self.rackdata.can_start_refresh(type, rack_name)
        self.map_progress.set_check_refresh_condition()
        if refresh_dict is None:
            return False
        
        total_hosts = {}
        if type == self.rackdata.TEMPERATURE_NAME:
            for rack_name in refresh_dict:
                if refresh_dict[rack_name]:
                    total_hosts[rack_name] = {"total": 0, "updated": 0, "ratio": 0}
                    # change the status of 'rack_status' to 'refresh'
                    self.rackdata.set_status_start_refresh(type, rack_name)
                    hosts = self.rackdata.inventory.hostlist(rack_name)
                    total_hosts[rack_name]["total"] = len(hosts)
                    self.map_progress.update_data("temperature_data", {rack_name: total_hosts[rack_name]})
                    # fetch data for each host
                    for server in hosts:
                        self.start_async_refresh(type, rack_name, server)
        elif type == self.rackdata.SERVICE_NAME:
            if reduce(lambda x, y: x or y, [refresh_dict[name] for name in refresh_dict], False):
                for rack_name in refresh_dict:
                    total_hosts[rack_name] = {"total": 0, "updated": 0, "ratio": 0}
                    # change the status of 'rack_status' to 'refresh'
                    self.rackdata.set_status_start_refresh(type, rack_name)
                    hosts = self.rackdata.inventory.hostlist(rack_name)
                    total_hosts[rack_name]["total"] = len(hosts)
                self.map_progress.update_data("service_data", total_hosts)
                # service refresh operation can update all rack ONLY once
                self.start_async_refresh(type, rack_name, None)
        
        self.rackdata.mydebug("exit from refresh_rack_data of {0}-{1} with refresh data {2}".format(rack_name, type, refresh_dict))
        
        return True
    
    
    # refresh rack temperature
    def refresh_rack_temperature(self, rack_name):
        return self.refresh_rack_data(self.rackdata.TEMPERATURE_NAME, rack_name)


    # refresh rack service
    def refresh_rack_service(self, rack_name):
        return self.refresh_rack_data(self.rackdata.SERVICE_NAME, rack_name)
    
    
    # API of generate map
    def start_gen_map(self, service, rack_name):
        t = threading.Thread(target=self.gen_map_thread, args=[service, rack_name])
        t.start()
        return True
    
    
    # API of refresh map
    # return True means start refresh process
    #        False means NOT start refresh process, "fresh" gives the reason
    #               if "fresh" is True, that means the data in db is fresh, does NOT need a refresh
    #                             Flase, means ERROR occured in db
    def start_refresh_map(self, service, rack_name):
        result = False
        flag_fresh = False
        refresh_dict = self.rackdata.can_start_refresh(service, rack_name)
        if refresh_dict:
            result = reduce(lambda x,y: x or y, [refresh_dict[name] for name in refresh_dict], False)
            if result:
                t = threading.Thread(target=self.refresh_map_thread, args=[service, rack_name])
                t.start()
            else:
                flag_fresh = True
            
        return {"result": result, "fresh": flag_fresh}
    
    
    # refresh map thread
    def refresh_map_thread(self, service, rack_name):
        self.get_map_progress(service)
        #self.map_progress.set_refresh_map()
        flag_data_ready = False
        if self.refresh_rack_data(service, rack_name):
            flag_data_ready = self.check_rack_refresh_status(service, rack_name)
        
        if flag_data_ready:
            self.rackwork.generate_map(service, rack_name, True)
        else:
            self.map_progress.set_error_status()    # refresh error, return to user
    
    
    # generate map thread
    def gen_map_thread(self, service, rack_name):
        self.get_map_progress(service)
        
        flag_read_refresh_data = False
        flag_data_ready = False
        
        if self.rackdata.is_rack_data_ready(service, rack_name):
            flag_data_ready = True
        else:
            self.map_progress.set_load_refresh_map()
            if self.refresh_rack_data(service, rack_name):
                flag_read_refresh_data = True
                flag_data_ready = self.check_rack_refresh_status(service, rack_name)
        
        if flag_data_ready:
            self.rackwork.generate_map(service, rack_name, flag_read_refresh_data)
        else:
            self.map_progress.set_error_status()    # read db error, return to user

        
    # update the status of async  data refresh
    def update_data_refresh_status(self, service):
        data_dict = self.map_progress.get_data("{0}_data".format(service))
        total_count = 0
        total_updated = 0
        for rack_name in data_dict:
            if data_dict[rack_name]["updated"] < data_dict[rack_name]["total"]:
                # query db to get recent update status
                query_dict = self.rackdata.get_rack_query_dict(service, rack_name, self.rackdata.LOCATION_TEMP)
                result_dict = self.rackdata.partly_query(query_dict[rack_name], {"updated_node": 1})
                data_dict[rack_name]["updated"] = result_dict["updated_node"]
            total_count += data_dict[rack_name]["total"]
            total_updated += data_dict[rack_name]["updated"]
        ratio = total_updated * 1.0 / total_count
        self.map_progress.set_async_refresh(round(ratio, 2))


    # check mongo db refresh status
    # default interval time is 200ms, and the max check count is 10000
    def check_rack_refresh_status(self, service, rack_name, interval=0.2, max_count=10000):
        result = False
        curr_count = 1
        while True:
            sleep(interval)  # default is 200 ms
            # check and update async data refresh status
            self.update_data_refresh_status(service)
            if self.rackdata.is_refresh_rack_data_ready(service, rack_name):
                self.map_progress.set_async_refresh()
                result = True
                break;
            curr_count += 1
            if curr_count > max_count:
                break;
        return result

    def get_map_progress(self, service):
        if service == self.rackdata.TEMPERATURE_NAME:
            self.map_progress = get_temperature_progress(self.username)
        elif service == self.rackdata.SERVICE_NAME:
            self.map_progress = get_service_progress(self.username)
        
        return self.map_progress