def run(arg):
cswithcer = codecSwitcher()
sConf = staticConfig()
path = sConf.getPath()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper( infoCLocation.get("ipInfoC"), infoCLocation.get("portInfoC"))
consumerLocation = arg[0]
stepSize = arg[1]
tagList = cswithcer.getEncode(arg[2:])
gchooser = groupChooser()
groupList = gchooser.chooseGroup(tagList)
if groupList == []:
print "No storage resource available"
print "failed1failed2failed"
sys.exit(1)
groupName = None
for groupN in groupList:
currentUsageInfo = getUsageInfo.run([])
gsize = currentUsageInfo.get(groupN).get('groupSize')
usize = currentUsageInfo.get(groupN).get('usedSize')
asize = long(gsize) - long(usize) - 100
ssize = long(stepSize)
if ssize <= asize:
groupName = groupN
break
if groupName == None:
print "Do not have enough storage space requestSize:%d and asize:%d" % (ssize,asize)
print "failed1failed2failed"
sys.exit(1)
requestStorageCmd = "ssh -t root@"+consumerLocation+" \"python "+path+"main.py requestStorage "+groupName+" "+str(stepSize)+" extra\""
out = executeCmd(requestStorageCmd)
if out.find("706errorKEY") >= 0:
print "failed1failed2failed"
sys.exit(1)
def __init__(self, args):
print args
self.cswitcher = codecSwitcher()
self.groupName = self.cswitcher.getEncode(args[0])
self.stepSize = args[1]
self.mode = args[2]
self.sConsumer = storageConsumer()
def __init__(self, source):
self.cswitcher = codecSwitcher()
self.logSource = source
self.logPath = "./suyiAutoscale.log"
# logging.basicConfig(filename=self.flogPath,filemode='a',level=logging.DEBUG,format='%(asctime)s - %(filename)s[func:%(funcName)s ; line:%(lineno)d] %(levelname)s\t: %(message)s')
# self.fLogger = logging.getLogger("root.fullLog")
logging.basicConfig(filename=self.logPath,filemode='a',level=logging.DEBUG,format='%(asctime)s - '+self.logSource+'\t: %(message)s')
def run(arg):
cswitcher = codecSwitcher()
deviceLocation = arg[0]
deviceName = arg[1]
groupName = cswitcher.getEncode(arg[2])
sConf = staticConfig()
path = sConf.getPath()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper(infoCLocation["ipInfoC"],infoCLocation["portInfoC"])
gmConf = cHelper.getGroupMConf().get(groupName)
if gmConf == None:
print "Group do not exist"
print "failed1failed2failed"
sys.exit(1)
providersConf = cHelper.getProviderConf()
providerID = deviceName+deviceLocation
for groupProvidersConf in providersConf.values():
if providerID in groupProvidersConf.keys():
print "Storage Device have been already added into system!"
print "failed1failed2failed"
sys.exit(1)
startProviderCmd = "ssh -t root@"+deviceLocation+" \"python "+path+"main.py startProvider "+deviceName+" "+groupName+"\""
out = executeCmd(startProviderCmd)
if out.find("706errorKEY") >= 0:
print "failed1failed2failed"
sys.exit(1)
gmIP = gmConf.get("gmIP")
extendGroupCmd = "ssh -t root@"+gmIP+" \"python "+path+"main.py extendGroup "+groupName+"\""
out = executeCmd(extendGroupCmd)
if out.find("706errorKEY") >= 0:
print "failed1failed2failed"
sys.exit(1)
def __init__(self,args):
print args
self.cswitcher = codecSwitcher()
self.groupName = self.cswitcher.getEncode(args[0])
self.groupManager = groupManager(self.groupName)
sConf = staticConfig()
infoCLocation = sConf.getInfoCLocation()
self.cHelper = configHelper(infoCLocation["ipInfoC"],infoCLocation["portInfoC"])
self.logger = autoscaleLog(__file__)
def run(arg):
os.environ['TZ']="Asia/Shanghai"
time.tzset()
cswitcher = codecSwitcher()
logger = autoscaleLog(__file__)
logger.writeLog(arg)
sConf = staticConfig()
path = sConf.getPath()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper( infoCLocation.get("ipInfoC"), infoCLocation.get("portInfoC"))
userName = arg[0]
stepSize = arg[1]
startTime = int(arg[2])
endTime = int(arg[3])
#tagList = []
#for a in arg[4:]:
# tagList.append(unicode(a))
tagList = cswitcher.getEncode(arg[4:])
gchooser = groupChooser()
groupList = gchooser.chooseGroup(tagList)
if groupList == []:
print "No storage resource available"
print "failed1failed2failed"
return False
groupName = groupList[0]
res = gchooser.applyTimeslot(groupName,startTime,endTime,stepSize)
if res == False:
print "Do not have enough space, booking failed"
print "failed1failed2failed"
return False
# update user booking table
userBooking = cHelper.getUserBookingTable()
print userBooking
userBookingForUser = userBooking.get(userName)
if userBookingForUser == None:
userBookingForUser = {}
userBookingForUserForGroup = userBookingForUser.get(groupName)
if userBookingForUserForGroup == None:
userBookingForUserForGroup = {}
stKey = startTime/3600
etKey = endTime/3600
for t in xrange(stKey,etKey):
s = userBookingForUserForGroup.get(str(t))
if s == None:
s = 0
userBookingForUserForGroup[str(t)] = s + int(stepSize)
userBookingForUser[groupName] = userBookingForUserForGroup
print userBookingForUserForGroup
userBooking[userName] = userBookingForUser
print userBookingForUser
print userBooking
cHelper.setUserBookingTable(userBooking)
print "booking succeded"
return True
def run(arg):
cswitcher = codecSwitcher()
logger = autoscaleLog(__file__)
logger.writeLog(sys.argv)
groupName = cswitcher.getEncode(arg[0])
#tagList = []
#for a in arg[1:]:
# tagList.append(unicode(a))
tagList = cswitcher.getEncode(arg[1:])
sConf = staticConfig()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper( infoCLocation.get("ipInfoC"), infoCLocation.get("portInfoC"))
providersConf = cHelper.getProviderConf()
if providersConf.get(groupName) == None:
# providersConf[groupName] = {"tags":tagList}
providersConf[groupName] = {}
else:
print "group exist"
print "failed1failed2failed"
sys.exit(1)
logger.writeLog(providersConf)
print json.dumps(providersConf)
gmConf = cHelper.getGroupMConf()
groupAmount = len(gmConf)
if gmConf.get(groupName) == None:
gmConf[groupName] = {}
gmConf[groupName]["currentTid"] = 500+200*groupAmount
gmConf[groupName]["gmIP"] = sConf.getGroupMIP()
gmConf[groupName]["devicesLoaded"] = []
gmConf[groupName]["consumersLoaded"] = []
else:
print "group exist"
print "failed1failed2failed"
sys.exit(1)
logger.writeLog(gmConf)
print json.dumps(gmConf)
tagsManager = cHelper.getTagsManager()
if tagsManager.get(groupName) == None:
tagsManager[groupName] = tagList
else:
print "group exist"
print "failed1failed2failed"
sys.exit(1)
cHelper.setProviderConf(providersConf)
cHelper.setGroupMConf(gmConf)
cHelper.setTagsManager(tagsManager)
logger.shutdownLog()
def __init__(self, groupName):
sConf = staticConfig()
self.ipInfoC = sConf.getInfoCLocation()["ipInfoC"]
self.portInfoC = sConf.getInfoCLocation()["portInfoC"]
self.cHelper = configHelper(self.ipInfoC,self.portInfoC)
hostName = self.executeCmd("hostname")
iframe = sConf.getHostInterface(hostName)
self.path = sConf.getPath()
self.hostIP = self.getLocalIP(iframe)
self.groupName = groupName
cswitcher = codecSwitcher()
self.groupNameHash = cswitcher.getHash(groupName)
self.vgName = self.groupNameHash + "VG"
self.loadConf()
for cmd in self.initialCmds:
self.executeCmd(cmd)
self.logger = autoscaleLog(__file__)
def run(arg):
cswitcher = codecSwitcher()
sConf = staticConfig()
path = sConf.getPath()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper(infoCLocation["ipInfoC"],infoCLocation["portInfoC"])
groupName = cswitcher.getEncode(arg[0])
gmsConf = cHelper.getGroupMConf()
gmConf = gmsConf.get(groupName)
if gmConf == None:
print "group do not exist"
print "failed1failed2failed"
return False
gmIP = gmConf.get("gmIP")
print gmIP
print path
print groupName
deleteGroupCmd = u"ssh -t root@"+gmIP+u" \"python "+path+u"main.py deleteGroup "+groupName+u"\""
out = executeCmd(deleteGroupCmd)
if out.find("706errorKEY") >= 0:
print "failed1failed2failed"
sys.exit(1)
def requestStorage(self,groupName,stepSize,mode):
extraDeviceConf = {}
remoteConsumersConf = self.cHelper.getConsumerConf()
remoteConsumerConf = remoteConsumersConf[self.conf["consumerID"]]
remoteGroupManagersConf = self.cHelper.getGroupMConf()
remoteGroupManagerConf = remoteGroupManagersConf[groupName]
remoteGroupManagerConf["currentTid"] += 1
remoteConsumerConf["remoteDiskAmount"] += 1
extraDeviceConf["remoteLV"] = remoteConsumerConf["remoteLV"]+str(remoteGroupManagerConf["currentTid"])
# extraDeviceConf["remoteLV"] = remoteConsumerConf["remoteLV"]+str(remoteConsumerConf["remoteDiskAmount"])
cswitcher = codecSwitcher()
extraDeviceConf["remoteVG"] = cswitcher.getHash(groupName)+"VG"
extraDeviceConf["remoteLVPath"] = "/dev/"+extraDeviceConf["remoteVG"]+"/"+extraDeviceConf["remoteLV"]
extraDeviceConf["remoteIQN"] = remoteConsumerConf["remoteIQN"]+str(remoteGroupManagerConf["currentTid"])
# extraDeviceConf["remoteIQN"] = remoteConsumerConf["remoteIQN"]+str(remoteConsumerConf["remoteDiskAmount"])
extraDeviceConf["groupName"] = groupName
extraDeviceConf["remoteSize"] = stepSize
extraDeviceConf["remoteTid"] = remoteGroupManagerConf["currentTid"]
groupManagerIP = remoteGroupManagerConf["gmIP"]
# createRemoteStorage
self.remoteCmd("lvcreate -L "+str(extraDeviceConf["remoteSize"])+"M -n "+extraDeviceConf["remoteLV"]+" "+extraDeviceConf["remoteVG"], groupManagerIP)
if self.iscsiTargetType == "tgt":
tgtStatus = self.remoteCmd("service tgtd status",groupManagerIP)
if "stopped" in tgtStatus:
self.remoteCmd("service tgtd start",groupManagerIP)
self.remoteCmd("tgtadm --lld iscsi --op new --mode target --tid "+str(extraDeviceConf["remoteTid"])+" -T "+extraDeviceConf["remoteIQN"] , groupManagerIP)
self.remoteCmd("setenforce 0;tgtadm --lld iscsi --op new --mode logicalunit --tid "+str(extraDeviceConf["remoteTid"])+" --lun 1 -b "+extraDeviceConf["remoteLVPath"] , groupManagerIP)
self.remoteCmd("tgtadm --lld iscsi --op bind --mode target --tid "+str(extraDeviceConf["remoteTid"])+" -I "+self.hostIP , groupManagerIP)
elif self.iscsiTargetType == "scst":
remoteIQN = extraDeviceConf["remoteIQN"]
remoteTid = str(extraDeviceConf["remoteTid"])
remoteLVPath = extraDeviceConf["remoteLVPath"]
cmdScstCreate = "scst-create.sh "+remoteIQN+" "+remoteTid+" "+remoteTid+" 0 "+remoteLVPath+" "+self.hostIP
self.remoteCmd(cmdScstCreate,groupManagerIP)
cmdScstRemove = "scst-remove.sh "+remoteIQN+" "+remoteTid+" "+remoteTid+" 0 "+remoteLVPath+" "+self.hostIP
self.remoteCmd("echo "+cmdScstRemove+" >/root/"+remoteTid+".sh", groupManagerIP)
else:
print "iscsi Target Type do not match"
print "706errorKEY"
sys.exit()
# loadRemoteStorage
baseDevices = self.getDevicesInfo()
cmdDiscovery = "iscsiadm -m discovery -t sendtargets -p " + groupManagerIP
self.executeCmd(cmdDiscovery)
cmdLogin = "******"+extraDeviceConf["remoteIQN"]+" --login"
self.executeCmd(cmdLogin)
time.sleep(5)
devices = self.getDevicesInfo()
newDevices = list(set(devices).difference(set(baseDevices)))
print "baseDevice"+str(baseDevices)
self.logger.writeLog("baseDevice"+str(baseDevices))
print "allDevice"+str(devices)
self.logger.writeLog("allDevice"+str(devices))
print "newDevice"+str(newDevices)
self.logger.writeLog("newDevice"+str(newDevices))
if newDevices == []:
print "Storage Load Failed"
print "706errorKEY"
self.logger.writeLog("Storage Load Failed")
sys.exit()
extraDeviceConf["localDeviceMap"] = newDevices
# update Information center
for d in newDevices:
consuemrInfo = {"consumerLocation":self.hostIP,"localDeviceMap":str(d)}
remoteGroupManagerConf["consumersLoaded"].append(consuemrInfo)
if mode == "extra":
releaseCandidateKey = str(d)+"@"+self.hostIP
releaseCandidates = self.cHelper.getReleaseCandidates()
if releaseCandidates == None:
releaseCandidates = {}
releaseCandidates[releaseCandidateKey] = int(stepSize)
self.cHelper.setReleaseCandidates(releaseCandidates)
self.cHelper.setGroupMConf(remoteGroupManagersConf)
remoteConsumerConf["extraDevicesList"].append(extraDeviceConf)
self.cHelper.setConsumerConf(remoteConsumersConf)
return newDevices
def __init__(self, args):
self.cswitcher = codecSwitcher()
self.groupName = self.cswitcher.getEncode(args[0])
print [self.groupName]
self.groupManager = groupManager(self.groupName)
self.logger = autoscaleLog(__file__)
def __init__(self,args):
print args
self.cswitcher = codecSwitcher()
self.deviceName = args[0]
self.groupName = self.cswitcher.getEncode(args[1])
self.sProvider = storageProvider(self.deviceName,self.groupName)
def run(arg):
cswitcher = codecSwitcher()
logger = autoscaleLog(__file__)
logger.writeLog(arg)
sConf = staticConfig()
path = sConf.getPath()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper( infoCLocation.get("ipInfoC"), infoCLocation.get("portInfoC"))
print arg
userName = arg[0]
consumerLocation = arg[1]
stepSize = int(arg[2])
tagList = cswitcher.getEncode(arg[3:])
gchooser = groupChooser()
groupList = gchooser.chooseGroup(tagList)
if groupList == []:
print "No storage resource available"
print "failed1failed2failed"
return False
groupName = groupList[0]
# currentTime = time.time()
timeM = timeManager()
currentTime = timeM.getTime()
# print "currentTime"
# print currentTime
ctKey = str(currentTime/3600)
# print "ctKey"
# print ctKey
userBooking = cHelper.getUserBookingTable()
# print "userBooking GET"
# print userBooking
userBookingForUser = userBooking.get(userName)
# print "userBookingForUser"
# print userBookingForUser
if userBookingForUser == None:
print "User did not book any storage"
print "failed1failed2failed"
return False
userBookingForUserForGroup = userBookingForUser.get(groupName)
# print "userBookingForUserForGroup"
# print userBookingForUserForGroup
if userBookingForUserForGroup == None:
print "User did not book storage for "+str(tagList)
print "failed1failed2failed"
return False
bookedStorageSize = userBookingForUserForGroup.get(ctKey)
# print "bookedStorageSize"
# print bookedStorageSize
if bookedStorageSize == None:
print "User did not book storage for this period"
print "failed1failed2failed"
return False
bookedStorageSize = int(bookedStorageSize)
if bookedStorageSize < stepSize:
print "User do not have enough booked storage space"
print "failed1failed2failed"
return False
# check current available space, if not enough , release
currentUsageInfo = getUsageInfo.run([])
groupSize = currentUsageInfo.get(groupName).get('groupSize')
usedSize = currentUsageInfo.get(groupName).get('usedSize')
asize = groupSize - usedSize
neededSize = 0
if stepSize > asize:
neededSize = stepSize - asize
if neededSize > 0:
releaseConsumerList = []
releaseCandidates = cHelper.getReleaseCandidates()
for deviceMapConsumerIP,size in releaseCandidates.items():
neededSize = neededSize - size
releaseConsumerList.append(deviceMapConsumerIP)
if neededSize <= 0:
break
if neededSize > 0:
print "Do not have enough space sorry"
return False
# release consumers
for deviceMapConsumerIP in releaseConsumerList:
deviceMap, consumerIP = deviceMapConsumerIP.split("@")
releaseExtraStorage.run([consumerIP,deviceMap])
requestStorageCmd = "ssh -t root@"+consumerLocation+" \"python "+path+"main.py requestStorage "+groupName+" "+str(stepSize)+" booked\""
print requestStorageCmd
out = executeCmd(requestStorageCmd)
if out.find("706errorKEY") >= 0:
print "failed1failed2failed"
sys.exit(1)
# userBookingForUserForGroup[ctKey] = bookedStorageSize - stepSize
startKeyInt = int(ctKey)
timeKeys = userBookingForUserForGroup.keys()
timeKeysInt = []
for timeKey in timeKeys:
timeKeysInt.append(int(timeKey))
timeKeysInt.sort()
for i in xrange(len(timeKeysInt)):
timeKeyInt = timeKeysInt[i]
if timeKeyInt < startKeyInt:
userBookingForUserForGroup.pop(cswitcher.getEncode(timeKeyInt))
elif timeKeyInt == startKeyInt:
timeKey = cswitcher.getEncode(timeKeyInt)
timeKeyBookedSize = userBookingForUserForGroup[timeKey]
userBookingForUserForGroup[timeKey] = timeKeyBookedSize - stepSize
startKeyInt += 1
else:
break
cHelper.setUserBookingTable(userBooking)
print "get booked storage succeded"
return True
