示例#1
0
    def getVolume(self, name_or_id, serv=None,  _user="", cached=False):
        self.Logger.debug("Entering with name_or_id=%s, serv=%s, cached=%s" % (name_or_id, serv, cached) )
        VolList = []
        if cached :
            if serv != None :
                serv_uuid=afs.LookupUtil[self._CFG.CELL_NAME].getFSUUID(serv,self._CFG,cached=True)
                # need function in util name_or_ip and name_or_id?
                if afsutil.isName(name_or_id) :
                    vol=self.DBManager.getFromCache(Volume,name=name_or_id,serv_uuid=serv_uuid)
                else :
                    vol=self.DBManager.getFromCache(Volume,vid=name_or_id,serv_uuid=serv_uuid)
                VolList.append(vol)
            else :
                if afsutil.isName(name_or_id) :
                    VolList=self.DBManager.getFromCache(Volume,name=name_or_id,mustBeUnique=False)
                else :
                    VolList=self.DBManager.getFromCache(Volume,vid=name_or_id,mustBeUnique=False)
                self.Logger.debug("OSDgetVolume: VolList=%s" % VolList)
            for vol in VolList :
                vol.ExtAttr=self.getExtVolAttr(vol.vid)
                vol.OsdAttr=self.DBManager.getFromCache(ExtVolAttr_OSD,vid=vol.vid)
            return VolList
        osdExtAttr=ExtVolAttr_OSD()
        odict=osdExtAttr.getDict()
        vdictList = self._osdvolDAO.getVolume(name_or_id, serv=serv, _cfg=self._CFG, _user=_user)
        self.Logger.debug("OSDgetVolume: vdictList=%s" % vdictList)
        for vdict in vdictList :
            StorageUsage=self.getStorageUsage([serv,],vdict["vid"])
            vdict["blocks_osd_on"]=StorageUsage["storageUsage"]["online"]["Data"]
            vdict["blocks_osd_off"]=StorageUsage["storageUsage"]["archival"]["Data"]
            vdict["blocks_fs"]=StorageUsage["storageUsage"]["fileserver"]["Data"]
            vdict["serv_uuid"]=afs.LookupUtil[self._CFG.CELL_NAME].getFSUUID(vdict["serv"],self._CFG,cached=False)
            vdict.pop("serv")
            self.Logger.debug("getVolume: vdict=%s" % vdict)
            # move stuff to OSD-dict
            for k in vdict.keys() :
                if k in odict.keys() : 
                    odict[k] = vdict[k] 
                    if k == "vid" : continue
                    vdict.pop(k)
            vol = Volume()
            # XXX: we need this to ignore the OsdAttr and ExtAttr in the copy-operation.
            vdict["ignAttrList"]=vol.ignAttrList
            vol.setByDict(vdict)
            if odict['osdPolicy'] != 0 : 
                vol.OsdAttr=odict
                osdExtAttr.setByDict(odict)

            if self._CFG.DB_CACHE :
                self.DBManager.setIntoCache(Volume,vol,vid=vol.vid,serv_uuid=serv_uuid,part=vol.part) 
                if odict['osdPolicy'] != 0 : 
                    self.DBManager.setIntoCache(ExtVolAttr_OSD,osdExtAttr,vid=osdExtAttr.vid) 
            VolList.append(vol) 
        return VolList
示例#2
0
    def getStorageUsage(self,servers,vid,oldStorageUsage=None,_user="") :
        """
        do a vos traverse on a single Volume and merge with another histogram, if given
        return result as a dict
        if cached = True, then save ExtVolAttr_OSD in DBCache
        """
        try:
            StorageUsage=self._osdvolDAO.traverse(servers, vid, _cfg=self._CFG, _user=_user)
        except:
            return emptyStorageUsageDict

        if oldStorageUsage == None : return StorageUsage
        if len(oldStorageUsage) == 0 : return StorageUsage

        # merge with given histogram
        # totals
        for k in StorageUsage["totals"].keys() :
            StorageUsage["totals"][k]["numFiles"] += oldStorageUsage["totals"][k]["numFiles"] 
            StorageUsage["totals"][k]["Data"] += oldStorageUsage["totals"][k]["Data"] 
        # logical
        for oldbin in oldStorageUsage["logical"] :
            found=False
            for b in range(len(StorageUsage["logical"])) :
                if oldbin["lowerSize"] == StorageUsage["logical"][b]["lowerSize"] :
                    found = True
                    StorageUsage["logical"][b]["numFiles"] += oldbin["numFiles"] 
                    StorageUsage["logical"][b]["binData"] += oldbin["binData"] 
                    break
            if not found :
                StorageUsage["logical"].append(oldbin)                
        # real storage
        for k in ["numFiles","Data"] :
            StorageUsage["storageUsage"]["fileserver"][k] += oldStorageUsage["storageUsage"]["fileserver"][k]
            StorageUsage["storageUsage"]["archival"][k] += oldStorageUsage["storageUsage"]["archival"][k]
            StorageUsage["storageUsage"]["online"][k] += oldStorageUsage["storageUsage"]["online"][k]
        # detailed 
        for k in oldStorageUsage["storageUsage"]["detailed"] :
            found = False
            for b in range(len(StorageUsage["storageUsage"]["detailed"])) :
                if StorageUsage["storageUsage"]["detailed"][b]["osdid"] == k["osdid"] :
                    found = True
                    StorageUsage["storageUsage"]["detailed"][b]["Data"] += k["Data"]
                    StorageUsage["storageUsage"]["detailed"][b]["numFiles"] += k["numFiles"]
            if not found :
                StorageUsage["storageUsage"]["detailed"].append(k)
        # withoutCopy
        for k in ["numFiles","Data"] :
            StorageUsage["withoutCopy"]["fileserver"][k] += oldStorageUsage["withoutCopy"]["fileserver"][k]
            StorageUsage["withoutCopy"]["archival"][k] += oldStorageUsage["withoutCopy"]["archival"][k]
            StorageUsage["withoutCopy"]["online"][k] += oldStorageUsage["withoutCopy"]["online"][k]
        for k in oldStorageUsage["storageUsage"]["detailed"] :
            found = False
            for b in range(len(StorageUsage["withoutCopy"]["detailed"])) :
                if StorageUsage["withoutCopy"]["detailed"][b]["osdid"] == k["osdid"] :
                    found = True
                    StorageUsage["withoutCopy"]["detailed"][b]["Data"] += k["Data"]
                    StorageUsage["withoutCopy"]["detailed"][b]["numFiles"] += k["numFiles"]
            if not found :
                StorageUsage["withoutCopy"]["detailed"].append(k)

        if self._CFG.DB_CACHE :
            OsdVolAttr={}
            OsdVolAttr['vid'] = vid
            OsdVolAttr['blocks_fs']=int(StorageUsage["storageUsage"]["fileserver"]["Data"])
            OsdVolAttr['blocks_osd_on']=int(StorageUsage["storageUsage"]["online"]["Data"])
            OsdVolAttr['blocks_osd_off']=int(StorageUsage["storageUsage"]["archival"]["Data"])
            thisObj=ExtVolAttr_OSD()
            thisObj.setByDict(OsdVolAttr)
            self.DBManager.setIntoCache(ExtVolAttr_OSD,thisObj,vid=thisObj.vid)
        return StorageUsage
示例#3
0
    def getServerSpread(self, prjname, cached = True):
        """
        return dict of lists of ProjectSpread Objects :
        [VolType]=[ProjectSpread]
        """
        thisProject=self.getProjectByName(prjname)
        if not thisProject : return None
        ResDict = {"RW" : [], "RO" : [], "BK" : []}

        if cached :
            for vol_type in ResDict :
                ResDict[vol_type]=self.DBManager.getFromCache(ProjectSpread, mustBeUnique=False, vol_type=vol_type, project_id = thisProject.id)
            return ResDict
              
        VolIDList = self.getVolumeIDs(prjname)
        if VolIDList == None : return None
        for v in VolIDList :
            VolGroup = self._VS.getVolGroup(v) 
            for vol_type in ResDict :
                if VolGroup[vol_type] == None : continue
                for vol in VolGroup[vol_type] :
                    FSUUID = vol.serv_uuid
                    Part = vol.part
                    OSDAttributes = self._VS.getExtVolAttr_OSD(vol.vid)
                    if OSDAttributes == None :
                        OSDAttributes = ExtVolAttr_OSD()
                        v = self._VS.getVolume(vol.vid,cached=True)
                        # satisfy OSD attributes
                        OSDAttributes.blocks_fs = v[0].diskused

                    thisPrjSPObj = None
                    ResDictIndex = -1
                    for i in range(len(ResDict[vol_type])) :
                        prjspObj = ResDict[vol_type][i]
                        self.Logger.debug("comparing %s,%s,%s with %s,%s,%s" % (prjspObj.serv_uuid, prjspObj.part, prjspObj.vol_type, FSUUID, Part, vol_type) )
                        if prjspObj.serv_uuid == FSUUID and prjspObj.part == Part and prjspObj.vol_type == vol_type :
                            thisPrjSPObj=prjspObj
                            ResDictIndex=i
                            break

                    if thisPrjSPObj == None :
                        thisPrjSPObj = ProjectSpread()
                        thisPrjSPObj.project_id = thisProject.id
                        thisPrjSPObj.part = Part
                        thisPrjSPObj.serv_uuid = FSUUID    
                        thisPrjSPObj.vol_type = vol_type
                    
                    thisPrjSPObj.num_vol += 1 
                    thisPrjSPObj.blocks_fs += OSDAttributes.blocks_fs
                    thisPrjSPObj.blocks_osd_on += OSDAttributes.blocks_osd_on
                    thisPrjSPObj.blocks_osd_off += OSDAttributes.blocks_osd_off
                    if ResDictIndex == -1 :
                        ResDict[vol_type].append(thisPrjSPObj)
                    else :
                        ResDict[vol_type][ResDictIndex] = thisPrjSPObj

        if self._CFG.DB_CACHE :
            for vol_type in ResDict :
                for thisPrjSPObj in ResDict[vol_type] :
                    self.DBManager.setIntoCache(ProjectSpread, thisPrjSPObj, vol_type=vol_type, project_id=thisPrjSPObj.project_id, serv_uuid=thisPrjSPObj.serv_uuid, part=thisPrjSPObj.part)
        return ResDict