def run(arg):
logger = autoscaleLog(__file__)
sConf = staticConfig()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper(infoCLocation["ipInfoC"],infoCLocation["portInfoC"])
path = sConf.getPath()
logFile = path+"suyiAutoscale.log"
try:
mode = arg.pop(0)
print mode
if mode == "status":
allInfo = {}
allInfo = getInfo.run([])
print json.dumps(allInfo,indent=1)
return
elif mode == "log":
logPath = arg.pop()
if logPath[-1] == "/":
logFile = logPath + "suyiAutoscale.log"
else:
logFile = logPath + "/suyiAutoscale.log"
print logFile
with open(logFile,"r") as log:
print log.read().replace('\\n','\n').replace('\\r','\r')
return
else:
print "Wrong mode"
return
except:
print 'wrong arguments'
return
def __init__(self):
self.staticConf["allNodes"] = ["192.168.0.98","192.168.0.97","192.168.0.99",
"192.168.16.121","192.168.16.122","192.168.16.123","192.168.16.124",
"192.168.16.101","192.168.16.102","192.168.16.103","192.168.16.104",
"192.168.16.105","192.168.16.106","192.168.1.88",
"192.168.16.111","192.168.16.112","192.168.16.113","192.168.16.114"]
self.staticConf["infoCLocation"] = {"ipInfoC":"192.168.1.88", "portInfoC":6379}
self.staticConf["path"] = "/opt/suyi/autoscale706kylin/"
self.staticConf["gmCandidates"] = ["192.168.16.105"]
self.staticConf["hostInterfaceMap"] = {"ds01":"eth0",
"ds02":"eth0",
"ds03":"eth0",
"ds04":"eth0",
"0-97":"bond0",
"0-98":"bond0",
"0-99":"bond0",
"client01":"eth3",
"client02":"eth3",
"client03":"eth9",
"client04":"eth5",
"client05":"eth2",
"client06":"eth3",
"mds01":"eth0",
"mds02":"eth0",
"mds03":"eth0",
"mds04":"eth0",
"jm":"eth0"}
self.staticConf["iscsiTargetType"] = {"groupManager":"tgt",
"storageProvider":"scst"} # another option is "tgt"
self.logger = autoscaleLog(__file__)
self.logger.writeLog(self.staticConf)
self.logger.shutdownLog()
def __init__(self):
os.environ['TZ']="Asia/Shanghai"
time.tzset()
self.logger = autoscaleLog(__file__)
self.sConf = staticConfig()
infoCLocation = self.sConf.getInfoCLocation()
self.cHelper = configHelper( infoCLocation.get("ipInfoC"), infoCLocation.get("portInfoC"))
self.getTime()
def executeCmd(cmd):
logger = autoscaleLog(__file__)
print cmd
logger.writeLog(cmd)
output = os.popen(cmd).read()
print output
logger.writeLog(output)
logger.shutdownLog()
return output
def __init__(self,args):
print args
self.cswitcher = codecSwitcher()
self.groupName = self.cswitcher.getEncode(args[0])
self.groupManager = groupManager(self.groupName)
sConf = staticConfig()
infoCLocation = sConf.getInfoCLocation()
self.cHelper = configHelper(infoCLocation["ipInfoC"],infoCLocation["portInfoC"])
self.logger = autoscaleLog(__file__)
def run(arg):
os.environ['TZ']="Asia/Shanghai"
time.tzset()
cswitcher = codecSwitcher()
logger = autoscaleLog(__file__)
logger.writeLog(arg)
sConf = staticConfig()
path = sConf.getPath()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper( infoCLocation.get("ipInfoC"), infoCLocation.get("portInfoC"))
userName = arg[0]
stepSize = arg[1]
startTime = int(arg[2])
endTime = int(arg[3])
#tagList = []
#for a in arg[4:]:
# tagList.append(unicode(a))
tagList = cswitcher.getEncode(arg[4:])
gchooser = groupChooser()
groupList = gchooser.chooseGroup(tagList)
if groupList == []:
print "No storage resource available"
print "failed1failed2failed"
return False
groupName = groupList[0]
res = gchooser.applyTimeslot(groupName,startTime,endTime,stepSize)
if res == False:
print "Do not have enough space, booking failed"
print "failed1failed2failed"
return False
# update user booking table
userBooking = cHelper.getUserBookingTable()
print userBooking
userBookingForUser = userBooking.get(userName)
if userBookingForUser == None:
userBookingForUser = {}
userBookingForUserForGroup = userBookingForUser.get(groupName)
if userBookingForUserForGroup == None:
userBookingForUserForGroup = {}
stKey = startTime/3600
etKey = endTime/3600
for t in xrange(stKey,etKey):
s = userBookingForUserForGroup.get(str(t))
if s == None:
s = 0
userBookingForUserForGroup[str(t)] = s + int(stepSize)
userBookingForUser[groupName] = userBookingForUserForGroup
print userBookingForUserForGroup
userBooking[userName] = userBookingForUser
print userBookingForUser
print userBooking
cHelper.setUserBookingTable(userBooking)
print "booking succeded"
return True
def __init__(self,arg):
self.userName = arg[0]
sConf = staticConfig()
self.logger = autoscaleLog(__file__)
self.ipInfoC = sConf.getInfoCLocation()["ipInfoC"]
self.portInfoC = sConf.getInfoCLocation()["portInfoC"]
self.cHelper = configHelper(self.ipInfoC,self.portInfoC)
hostName = self.executeCmd("hostname")
iframe = sConf.getHostInterface(hostName)
self.hostIP = self.getLocalIP(iframe)
def executeCmd(cmd):
logger = autoscaleLog(__file__)
print cmd
#cmd = unicode(cmd,'utf-8')
print [cmd]
output = executeCmdSp(cmd)
print output
logger.writeLog(cmd)
logger.writeLog(output)
logger.shutdownLog()
return output
def __init__(self):
self.logger = autoscaleLog(__file__)
sConf = staticConfig()
self.ipInfoC = sConf.getInfoCLocation()["ipInfoC"]
self.portInfoC = sConf.getInfoCLocation()["portInfoC"]
self.cHelper = configHelper(self.ipInfoC,self.portInfoC)
hostName = self.executeCmd("hostname")
iframe = sConf.getHostInterface(hostName)
self.hostIP = self.getLocalIP(iframe)
self.loadConf()
self.iscsiTargetType = sConf.getTargetType("groupManager")
for cmd in self.initialCmds:
self.executeCmd(cmd)
def run(arg):
logger = autoscaleLog(__file__)
sConf = staticConfig()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper(infoCLocation["ipInfoC"],infoCLocation["portInfoC"])
consumersConf = cHelper.getConsumerConf()
providersConf = cHelper.getProviderConf()
tagsManager = cHelper.getTagsManager()
allConf = {}
allConf["dev"] = providersConf
allConf["appserver"] = consumersConf
allConf["tags"] = tagsManager
logger.writeLog(allConf)
print json.dumps(allConf)
logger.shutdownLog()
return allConf
def run(arg):
cswitcher = codecSwitcher()
logger = autoscaleLog(__file__)
logger.writeLog(sys.argv)
groupName = cswitcher.getEncode(arg[0])
#tagList = []
#for a in arg[1:]:
# tagList.append(unicode(a))
tagList = cswitcher.getEncode(arg[1:])
sConf = staticConfig()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper( infoCLocation.get("ipInfoC"), infoCLocation.get("portInfoC"))
providersConf = cHelper.getProviderConf()
if providersConf.get(groupName) == None:
# providersConf[groupName] = {"tags":tagList}
providersConf[groupName] = {}
else:
print "group exist"
print "failed1failed2failed"
sys.exit(1)
logger.writeLog(providersConf)
print json.dumps(providersConf)
gmConf = cHelper.getGroupMConf()
groupAmount = len(gmConf)
if gmConf.get(groupName) == None:
gmConf[groupName] = {}
gmConf[groupName]["currentTid"] = 500+200*groupAmount
gmConf[groupName]["gmIP"] = sConf.getGroupMIP()
gmConf[groupName]["devicesLoaded"] = []
gmConf[groupName]["consumersLoaded"] = []
else:
print "group exist"
print "failed1failed2failed"
sys.exit(1)
logger.writeLog(gmConf)
print json.dumps(gmConf)
tagsManager = cHelper.getTagsManager()
if tagsManager.get(groupName) == None:
tagsManager[groupName] = tagList
else:
print "group exist"
print "failed1failed2failed"
sys.exit(1)
cHelper.setProviderConf(providersConf)
cHelper.setGroupMConf(gmConf)
cHelper.setTagsManager(tagsManager)
logger.shutdownLog()
def __init__(self, groupName):
sConf = staticConfig()
self.ipInfoC = sConf.getInfoCLocation()["ipInfoC"]
self.portInfoC = sConf.getInfoCLocation()["portInfoC"]
self.cHelper = configHelper(self.ipInfoC,self.portInfoC)
hostName = self.executeCmd("hostname")
iframe = sConf.getHostInterface(hostName)
self.path = sConf.getPath()
self.hostIP = self.getLocalIP(iframe)
self.groupName = groupName
cswitcher = codecSwitcher()
self.groupNameHash = cswitcher.getHash(groupName)
self.vgName = self.groupNameHash + "VG"
self.loadConf()
for cmd in self.initialCmds:
self.executeCmd(cmd)
self.logger = autoscaleLog(__file__)
def run(arg):
username = arg[0]
logger = autoscaleLog(__file__)
sConf = staticConfig()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper(infoCLocation["ipInfoC"], infoCLocation["portInfoC"])
consumersConf = cHelper.getConsumerConf()
userConsumers = cHelper.getUserConsumers()
consumers = userConsumers.get(username)
if consumers == None:
consumers = []
userConsumersConf = {}
for consumer in consumers:
conf = consumersConf.get(consumer)
if conf != None:
userConsumersConf[consumer] = conf
logger.writeLog(userConsumersConf)
print json.dumps(userConsumersConf)
logger.shutdownLog()
def __init__(self,deviceName,groupName):
sConf = staticConfig()
self.ipInfoC = sConf.getInfoCLocation()["ipInfoC"]
self.iscsiTargetType = sConf.getTargetType("storageProvider")
self.portInfoC = sConf.getInfoCLocation()["portInfoC"]
self.cHelper = configHelper(self.ipInfoC,self.portInfoC)
hostName = self.executeCmd("hostname")
iframe = sConf.getHostInterface(hostName)
self.hostIP = self.getLocalIP(iframe)
self.conf["deviceName"] = deviceName
self.conf["deviceGroup"] = groupName
self.loadConf()
for cmd in self.initialCmds:
self.executeCmd(cmd)
self.logger = autoscaleLog(__file__)
sConf = staticConfig()
self.path = sConf.getPath()+"core/"
confPath = sConf.getPath()+"conf/storageProviderConf.json"
with open(confPath,"r") as fspc:
self.storageProviderConf = json.loads(fspc.read())
def __init__(self, args):
self.cswitcher = codecSwitcher()
self.groupName = self.cswitcher.getEncode(args[0])
print [self.groupName]
self.groupManager = groupManager(self.groupName)
self.logger = autoscaleLog(__file__)
def __init__(self):
self.logger = autoscaleLog(__file__)
self.sConf = staticConfig()
infoCLocation = self.sConf.getInfoCLocation()
self.cHelper = configHelper( infoCLocation.get("ipInfoC"), infoCLocation.get("portInfoC"))
def run(arg):
logger = autoscaleLog(__file__)
sConf = staticConfig()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper(infoCLocation["ipInfoC"], infoCLocation["portInfoC"])
usersBooking = cHelper.getUserBookingTable()
print usersBooking
tagManager = cHelper.getTagsManager()
userName = unicode(arg[0], "utf-8")
print userName
userBooking = usersBooking.get(userName)
print userBooking
userBookingRecords = []
if userBooking == None:
print json.dumps(userBookingRecords)
return userBookingRecords
for group, bookingTable in userBooking.items():
tags = tagManager.get(group)
userBookingForGroup = userBooking.get(group)
timestamps = []
for t in userBookingForGroup.keys():
timestamps.append(int(t))
timestamps.sort()
print timestamps
try:
bookingRecord = {}
startT = timestamps[0]
startS = userBookingForGroup.get(str(startT))
endT = None
for i in xrange(len(timestamps)):
middleT = timestamps[i]
endT = middleT
middleS = userBookingForGroup.get(str(middleT))
print middleT, middleS, timestamps[i + 1]
print endT
print middleT != (timestamps[i + 1] - 1)
print middleS != userBookingForGroup.get(str(timestamps[i + 1]))
print (middleT != (timestamps[i + 1] - 1)) or (
middleS != userBookingForGroup.get(str(timestamps[i + 1]))
)
if (middleT != (timestamps[i + 1] - 1)) or (middleS != userBookingForGroup.get(str(timestamps[i + 1]))):
bookingRecord["StartTime"] = time.ctime(startT * 3600)
bookingRecord["EndTime"] = time.ctime(endT * 3600 + 3600)
bookingRecord["Size"] = "%s MB" % (str(startS))
bookingRecord["User"] = userName
bookingRecord["Tag1"] = tags[0]
bookingRecord["Tag2"] = tags[1]
userBookingRecords.append(bookingRecord)
bookingRecord = {}
startT = timestamps[i + 1]
startS = userBookingForGroup.get(str(startT))
except:
bookingRecord["StartTime"] = time.ctime(startT * 3600)
bookingRecord["EndTime"] = time.ctime(endT * 3600 + 3600)
bookingRecord["Size"] = "%s MB" % (str(startS))
bookingRecord["User"] = userName
bookingRecord["Tag1"] = tags[0]
bookingRecord["Tag2"] = tags[1]
userBookingRecords.append(bookingRecord)
print json.dumps(userBookingRecords)
return userBookingRecords
print json.dumps(userBookingRecords)
return userBookingRecords
def run(arg):
cswitcher = codecSwitcher()
logger = autoscaleLog(__file__)
logger.writeLog(arg)
sConf = staticConfig()
path = sConf.getPath()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper( infoCLocation.get("ipInfoC"), infoCLocation.get("portInfoC"))
print arg
userName = arg[0]
consumerLocation = arg[1]
stepSize = int(arg[2])
tagList = cswitcher.getEncode(arg[3:])
gchooser = groupChooser()
groupList = gchooser.chooseGroup(tagList)
if groupList == []:
print "No storage resource available"
print "failed1failed2failed"
return False
groupName = groupList[0]
# currentTime = time.time()
timeM = timeManager()
currentTime = timeM.getTime()
# print "currentTime"
# print currentTime
ctKey = str(currentTime/3600)
# print "ctKey"
# print ctKey
userBooking = cHelper.getUserBookingTable()
# print "userBooking GET"
# print userBooking
userBookingForUser = userBooking.get(userName)
# print "userBookingForUser"
# print userBookingForUser
if userBookingForUser == None:
print "User did not book any storage"
print "failed1failed2failed"
return False
userBookingForUserForGroup = userBookingForUser.get(groupName)
# print "userBookingForUserForGroup"
# print userBookingForUserForGroup
if userBookingForUserForGroup == None:
print "User did not book storage for "+str(tagList)
print "failed1failed2failed"
return False
bookedStorageSize = userBookingForUserForGroup.get(ctKey)
# print "bookedStorageSize"
# print bookedStorageSize
if bookedStorageSize == None:
print "User did not book storage for this period"
print "failed1failed2failed"
return False
bookedStorageSize = int(bookedStorageSize)
if bookedStorageSize < stepSize:
print "User do not have enough booked storage space"
print "failed1failed2failed"
return False
# check current available space, if not enough , release
currentUsageInfo = getUsageInfo.run([])
groupSize = currentUsageInfo.get(groupName).get('groupSize')
usedSize = currentUsageInfo.get(groupName).get('usedSize')
asize = groupSize - usedSize
neededSize = 0
if stepSize > asize:
neededSize = stepSize - asize
if neededSize > 0:
releaseConsumerList = []
releaseCandidates = cHelper.getReleaseCandidates()
for deviceMapConsumerIP,size in releaseCandidates.items():
neededSize = neededSize - size
releaseConsumerList.append(deviceMapConsumerIP)
if neededSize <= 0:
break
if neededSize > 0:
print "Do not have enough space sorry"
return False
# release consumers
for deviceMapConsumerIP in releaseConsumerList:
deviceMap, consumerIP = deviceMapConsumerIP.split("@")
releaseExtraStorage.run([consumerIP,deviceMap])
requestStorageCmd = "ssh -t root@"+consumerLocation+" \"python "+path+"main.py requestStorage "+groupName+" "+str(stepSize)+" booked\""
print requestStorageCmd
out = executeCmd(requestStorageCmd)
if out.find("706errorKEY") >= 0:
print "failed1failed2failed"
sys.exit(1)
# userBookingForUserForGroup[ctKey] = bookedStorageSize - stepSize
startKeyInt = int(ctKey)
timeKeys = userBookingForUserForGroup.keys()
timeKeysInt = []
for timeKey in timeKeys:
timeKeysInt.append(int(timeKey))
timeKeysInt.sort()
for i in xrange(len(timeKeysInt)):
timeKeyInt = timeKeysInt[i]
if timeKeyInt < startKeyInt:
userBookingForUserForGroup.pop(cswitcher.getEncode(timeKeyInt))
elif timeKeyInt == startKeyInt:
timeKey = cswitcher.getEncode(timeKeyInt)
timeKeyBookedSize = userBookingForUserForGroup[timeKey]
userBookingForUserForGroup[timeKey] = timeKeyBookedSize - stepSize
startKeyInt += 1
else:
break
cHelper.setUserBookingTable(userBooking)
print "get booked storage succeded"
return True