def getVolume(self, name_or_id, serv=None, _user="", cached=False):
self.Logger.debug("Entering with name_or_id=%s, serv=%s, cached=%s" % (name_or_id, serv, cached) )
VolList = []
if cached :
if serv != None :
serv_uuid=afs.LookupUtil[self._CFG.CELL_NAME].getFSUUID(serv,self._CFG,cached=True)
# need function in util name_or_ip and name_or_id?
if afsutil.isName(name_or_id) :
vol=self.DBManager.getFromCache(Volume,name=name_or_id,serv_uuid=serv_uuid)
else :
vol=self.DBManager.getFromCache(Volume,vid=name_or_id,serv_uuid=serv_uuid)
VolList.append(vol)
else :
if afsutil.isName(name_or_id) :
VolList=self.DBManager.getFromCache(Volume,name=name_or_id,mustBeUnique=False)
else :
VolList=self.DBManager.getFromCache(Volume,vid=name_or_id,mustBeUnique=False)
self.Logger.debug("OSDgetVolume: VolList=%s" % VolList)
for vol in VolList :
vol.ExtAttr=self.getExtVolAttr(vol.vid)
vol.OsdAttr=self.DBManager.getFromCache(ExtVolAttr_OSD,vid=vol.vid)
return VolList
osdExtAttr=ExtVolAttr_OSD()
odict=osdExtAttr.getDict()
vdictList = self._osdvolDAO.getVolume(name_or_id, serv=serv, _cfg=self._CFG, _user=_user)
self.Logger.debug("OSDgetVolume: vdictList=%s" % vdictList)
for vdict in vdictList :
StorageUsage=self.getStorageUsage([serv,],vdict["vid"])
vdict["blocks_osd_on"]=StorageUsage["storageUsage"]["online"]["Data"]
vdict["blocks_osd_off"]=StorageUsage["storageUsage"]["archival"]["Data"]
vdict["blocks_fs"]=StorageUsage["storageUsage"]["fileserver"]["Data"]
vdict["serv_uuid"]=afs.LookupUtil[self._CFG.CELL_NAME].getFSUUID(vdict["serv"],self._CFG,cached=False)
vdict.pop("serv")
self.Logger.debug("getVolume: vdict=%s" % vdict)
# move stuff to OSD-dict
for k in vdict.keys() :
if k in odict.keys() :
odict[k] = vdict[k]
if k == "vid" : continue
vdict.pop(k)
vol = Volume()
# XXX: we need this to ignore the OsdAttr and ExtAttr in the copy-operation.
vdict["ignAttrList"]=vol.ignAttrList
vol.setByDict(vdict)
if odict['osdPolicy'] != 0 :
vol.OsdAttr=odict
osdExtAttr.setByDict(odict)
if self._CFG.DB_CACHE :
self.DBManager.setIntoCache(Volume,vol,vid=vol.vid,serv_uuid=serv_uuid,part=vol.part)
if odict['osdPolicy'] != 0 :
self.DBManager.setIntoCache(ExtVolAttr_OSD,osdExtAttr,vid=osdExtAttr.vid)
VolList.append(vol)
return VolList
def getStorageUsage(self,servers,vid,oldStorageUsage=None,_user="") :
"""
do a vos traverse on a single Volume and merge with another histogram, if given
return result as a dict
if cached = True, then save ExtVolAttr_OSD in DBCache
"""
try:
StorageUsage=self._osdvolDAO.traverse(servers, vid, _cfg=self._CFG, _user=_user)
except:
return emptyStorageUsageDict
if oldStorageUsage == None : return StorageUsage
if len(oldStorageUsage) == 0 : return StorageUsage
# merge with given histogram
# totals
for k in StorageUsage["totals"].keys() :
StorageUsage["totals"][k]["numFiles"] += oldStorageUsage["totals"][k]["numFiles"]
StorageUsage["totals"][k]["Data"] += oldStorageUsage["totals"][k]["Data"]
# logical
for oldbin in oldStorageUsage["logical"] :
found=False
for b in range(len(StorageUsage["logical"])) :
if oldbin["lowerSize"] == StorageUsage["logical"][b]["lowerSize"] :
found = True
StorageUsage["logical"][b]["numFiles"] += oldbin["numFiles"]
StorageUsage["logical"][b]["binData"] += oldbin["binData"]
break
if not found :
StorageUsage["logical"].append(oldbin)
# real storage
for k in ["numFiles","Data"] :
StorageUsage["storageUsage"]["fileserver"][k] += oldStorageUsage["storageUsage"]["fileserver"][k]
StorageUsage["storageUsage"]["archival"][k] += oldStorageUsage["storageUsage"]["archival"][k]
StorageUsage["storageUsage"]["online"][k] += oldStorageUsage["storageUsage"]["online"][k]
# detailed
for k in oldStorageUsage["storageUsage"]["detailed"] :
found = False
for b in range(len(StorageUsage["storageUsage"]["detailed"])) :
if StorageUsage["storageUsage"]["detailed"][b]["osdid"] == k["osdid"] :
found = True
StorageUsage["storageUsage"]["detailed"][b]["Data"] += k["Data"]
StorageUsage["storageUsage"]["detailed"][b]["numFiles"] += k["numFiles"]
if not found :
StorageUsage["storageUsage"]["detailed"].append(k)
# withoutCopy
for k in ["numFiles","Data"] :
StorageUsage["withoutCopy"]["fileserver"][k] += oldStorageUsage["withoutCopy"]["fileserver"][k]
StorageUsage["withoutCopy"]["archival"][k] += oldStorageUsage["withoutCopy"]["archival"][k]
StorageUsage["withoutCopy"]["online"][k] += oldStorageUsage["withoutCopy"]["online"][k]
for k in oldStorageUsage["storageUsage"]["detailed"] :
found = False
for b in range(len(StorageUsage["withoutCopy"]["detailed"])) :
if StorageUsage["withoutCopy"]["detailed"][b]["osdid"] == k["osdid"] :
found = True
StorageUsage["withoutCopy"]["detailed"][b]["Data"] += k["Data"]
StorageUsage["withoutCopy"]["detailed"][b]["numFiles"] += k["numFiles"]
if not found :
StorageUsage["withoutCopy"]["detailed"].append(k)
if self._CFG.DB_CACHE :
OsdVolAttr={}
OsdVolAttr['vid'] = vid
OsdVolAttr['blocks_fs']=int(StorageUsage["storageUsage"]["fileserver"]["Data"])
OsdVolAttr['blocks_osd_on']=int(StorageUsage["storageUsage"]["online"]["Data"])
OsdVolAttr['blocks_osd_off']=int(StorageUsage["storageUsage"]["archival"]["Data"])
thisObj=ExtVolAttr_OSD()
thisObj.setByDict(OsdVolAttr)
self.DBManager.setIntoCache(ExtVolAttr_OSD,thisObj,vid=thisObj.vid)
return StorageUsage
def getServerSpread(self, prjname, cached = True):
"""
return dict of lists of ProjectSpread Objects :
[VolType]=[ProjectSpread]
"""
thisProject=self.getProjectByName(prjname)
if not thisProject : return None
ResDict = {"RW" : [], "RO" : [], "BK" : []}
if cached :
for vol_type in ResDict :
ResDict[vol_type]=self.DBManager.getFromCache(ProjectSpread, mustBeUnique=False, vol_type=vol_type, project_id = thisProject.id)
return ResDict
VolIDList = self.getVolumeIDs(prjname)
if VolIDList == None : return None
for v in VolIDList :
VolGroup = self._VS.getVolGroup(v)
for vol_type in ResDict :
if VolGroup[vol_type] == None : continue
for vol in VolGroup[vol_type] :
FSUUID = vol.serv_uuid
Part = vol.part
OSDAttributes = self._VS.getExtVolAttr_OSD(vol.vid)
if OSDAttributes == None :
OSDAttributes = ExtVolAttr_OSD()
v = self._VS.getVolume(vol.vid,cached=True)
# satisfy OSD attributes
OSDAttributes.blocks_fs = v[0].diskused
thisPrjSPObj = None
ResDictIndex = -1
for i in range(len(ResDict[vol_type])) :
prjspObj = ResDict[vol_type][i]
self.Logger.debug("comparing %s,%s,%s with %s,%s,%s" % (prjspObj.serv_uuid, prjspObj.part, prjspObj.vol_type, FSUUID, Part, vol_type) )
if prjspObj.serv_uuid == FSUUID and prjspObj.part == Part and prjspObj.vol_type == vol_type :
thisPrjSPObj=prjspObj
ResDictIndex=i
break
if thisPrjSPObj == None :
thisPrjSPObj = ProjectSpread()
thisPrjSPObj.project_id = thisProject.id
thisPrjSPObj.part = Part
thisPrjSPObj.serv_uuid = FSUUID
thisPrjSPObj.vol_type = vol_type
thisPrjSPObj.num_vol += 1
thisPrjSPObj.blocks_fs += OSDAttributes.blocks_fs
thisPrjSPObj.blocks_osd_on += OSDAttributes.blocks_osd_on
thisPrjSPObj.blocks_osd_off += OSDAttributes.blocks_osd_off
if ResDictIndex == -1 :
ResDict[vol_type].append(thisPrjSPObj)
else :
ResDict[vol_type][ResDictIndex] = thisPrjSPObj
if self._CFG.DB_CACHE :
for vol_type in ResDict :
for thisPrjSPObj in ResDict[vol_type] :
self.DBManager.setIntoCache(ProjectSpread, thisPrjSPObj, vol_type=vol_type, project_id=thisPrjSPObj.project_id, serv_uuid=thisPrjSPObj.serv_uuid, part=thisPrjSPObj.part)
return ResDict
