def run(arg):
cswitcher = codecSwitcher()
deviceLocation = arg[0]
deviceName = arg[1]
groupName = cswitcher.getEncode(arg[2])
sConf = staticConfig()
path = sConf.getPath()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper(infoCLocation["ipInfoC"],infoCLocation["portInfoC"])
gmConf = cHelper.getGroupMConf().get(groupName)
if gmConf == None:
print "Group do not exist"
print "failed1failed2failed"
sys.exit(1)
providersConf = cHelper.getProviderConf()
providerID = deviceName+deviceLocation
for groupProvidersConf in providersConf.values():
if providerID in groupProvidersConf.keys():
print "Storage Device have been already added into system!"
print "failed1failed2failed"
sys.exit(1)
startProviderCmd = "ssh -t root@"+deviceLocation+" \"python "+path+"main.py startProvider "+deviceName+" "+groupName+"\""
out = executeCmd(startProviderCmd)
if out.find("706errorKEY") >= 0:
print "failed1failed2failed"
sys.exit(1)
gmIP = gmConf.get("gmIP")
extendGroupCmd = "ssh -t root@"+gmIP+" \"python "+path+"main.py extendGroup "+groupName+"\""
out = executeCmd(extendGroupCmd)
if out.find("706errorKEY") >= 0:
print "failed1failed2failed"
sys.exit(1)
def run(arg):
logger = autoscaleLog(__file__)
sConf = staticConfig()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper(infoCLocation["ipInfoC"],infoCLocation["portInfoC"])
path = sConf.getPath()
logFile = path+"suyiAutoscale.log"
try:
mode = arg.pop(0)
print mode
if mode == "status":
allInfo = {}
allInfo = getInfo.run([])
print json.dumps(allInfo,indent=1)
return
elif mode == "log":
logPath = arg.pop()
if logPath[-1] == "/":
logFile = logPath + "suyiAutoscale.log"
else:
logFile = logPath + "/suyiAutoscale.log"
print logFile
with open(logFile,"r") as log:
print log.read().replace('\\n','\n').replace('\\r','\r')
return
else:
print "Wrong mode"
return
except:
print 'wrong arguments'
return
def run(arg):
userName = arg[0]
consumerLocation = arg[1]
sConf = staticConfig()
path = sConf.getPath()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper( infoCLocation.get("ipInfoC"), infoCLocation.get("portInfoC"))
consumersConf = cHelper.getConsumerConf()
if consumerLocation in consumersConf.keys():
print "consumer already exist!"
print "failed1failed2failed"
sys.exit(1)
cmd = "ssh -t root@"+consumerLocation+" \"python "+path+"main.py startConsumer "+userName+"\""
userConsumers = cHelper.getUserConsumers()
# for uConsumers in userConsumers.values():
# if consumerLocation in uConsumers:
# print "consumer already exist!"
# print "failed1failed2failed"
# sys.exit(1)
userC = userConsumers.get(userName)
if userC == None:
userC = []
# if consumerLocation in userC:
# print "consumer already exist!"
# print "failed1failed2failed"
# return False
userC.append(consumerLocation)
userConsumers[userName] = userC
cHelper.setUserConsumers(userConsumers)
out = executeCmd(cmd)
if out.find("706errorKEY") >= 0:
print "failed1failed2failed"
sys.exit(1)
def run(arg):
cswithcer = codecSwitcher()
sConf = staticConfig()
path = sConf.getPath()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper( infoCLocation.get("ipInfoC"), infoCLocation.get("portInfoC"))
consumerLocation = arg[0]
stepSize = arg[1]
tagList = cswithcer.getEncode(arg[2:])
gchooser = groupChooser()
groupList = gchooser.chooseGroup(tagList)
if groupList == []:
print "No storage resource available"
print "failed1failed2failed"
sys.exit(1)
groupName = None
for groupN in groupList:
currentUsageInfo = getUsageInfo.run([])
gsize = currentUsageInfo.get(groupN).get('groupSize')
usize = currentUsageInfo.get(groupN).get('usedSize')
asize = long(gsize) - long(usize) - 100
ssize = long(stepSize)
if ssize <= asize:
groupName = groupN
break
if groupName == None:
print "Do not have enough storage space requestSize:%d and asize:%d" % (ssize,asize)
print "failed1failed2failed"
sys.exit(1)
requestStorageCmd = "ssh -t root@"+consumerLocation+" \"python "+path+"main.py requestStorage "+groupName+" "+str(stepSize)+" extra\""
out = executeCmd(requestStorageCmd)
if out.find("706errorKEY") >= 0:
print "failed1failed2failed"
sys.exit(1)
def run(arg):
consumerLocation = arg[0]
sConf = staticConfig()
path = sConf.getPath()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper(infoCLocation.get("ipInfoC"), infoCLocation.get("portInfoC"))
consumersConf = cHelper.getConsumerConf()
consumerConf = consumersConf.get(consumerLocation)
for localDeviceConf in consumerConf.get("extraDevicesList"):
localDeviceMap = localDeviceConf.get("localDeviceMap")
if localDeviceMap == None:
continue
localDeviceMap = localDeviceMap.pop(0)
if localDeviceMap == None:
continue
releaseStorageCmd = (
"ssh -t root@" + consumerLocation + ' "python ' + path + "main.py releaseStorage " + localDeviceMap + '"'
)
out = executeCmd(releaseStorageCmd)
if out.find("706errorKEY") >= 0:
print "failed1failed2failed"
sys.exit(1)
del consumersConf[consumerLocation]
cHelper.setConsumerConf(consumersConf)
userConsumers = cHelper.getUserConsumers()
for user, uConsumers in userConsumers.items():
if consumerLocation in uConsumers:
ucs = uConsumers
ucs.remove(consumerLocation)
userConsumers[user] = ucs
cHelper.setUserConsumers(userConsumers)
def __init__(self):
os.environ['TZ']="Asia/Shanghai"
time.tzset()
self.logger = autoscaleLog(__file__)
self.sConf = staticConfig()
infoCLocation = self.sConf.getInfoCLocation()
self.cHelper = configHelper( infoCLocation.get("ipInfoC"), infoCLocation.get("portInfoC"))
self.getTime()
def __init__(self,args):
print args
self.cswitcher = codecSwitcher()
self.groupName = self.cswitcher.getEncode(args[0])
self.groupManager = groupManager(self.groupName)
sConf = staticConfig()
infoCLocation = sConf.getInfoCLocation()
self.cHelper = configHelper(infoCLocation["ipInfoC"],infoCLocation["portInfoC"])
self.logger = autoscaleLog(__file__)
def run(self):
sConf = staticConfig()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper( infoCLocation.get("ipInfoC"), infoCLocation.get("portInfoC"))
newDevices = self.sConsumer.requestStorage(self.groupName,self.stepSize,self.mode)
if newDevices == []:
print "706errorKEY"
return
print str(newDevices)
def run(arg):
os.environ['TZ']="Asia/Shanghai"
time.tzset()
cswitcher = codecSwitcher()
logger = autoscaleLog(__file__)
logger.writeLog(arg)
sConf = staticConfig()
path = sConf.getPath()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper( infoCLocation.get("ipInfoC"), infoCLocation.get("portInfoC"))
userName = arg[0]
stepSize = arg[1]
startTime = int(arg[2])
endTime = int(arg[3])
#tagList = []
#for a in arg[4:]:
# tagList.append(unicode(a))
tagList = cswitcher.getEncode(arg[4:])
gchooser = groupChooser()
groupList = gchooser.chooseGroup(tagList)
if groupList == []:
print "No storage resource available"
print "failed1failed2failed"
return False
groupName = groupList[0]
res = gchooser.applyTimeslot(groupName,startTime,endTime,stepSize)
if res == False:
print "Do not have enough space, booking failed"
print "failed1failed2failed"
return False
# update user booking table
userBooking = cHelper.getUserBookingTable()
print userBooking
userBookingForUser = userBooking.get(userName)
if userBookingForUser == None:
userBookingForUser = {}
userBookingForUserForGroup = userBookingForUser.get(groupName)
if userBookingForUserForGroup == None:
userBookingForUserForGroup = {}
stKey = startTime/3600
etKey = endTime/3600
for t in xrange(stKey,etKey):
s = userBookingForUserForGroup.get(str(t))
if s == None:
s = 0
userBookingForUserForGroup[str(t)] = s + int(stepSize)
userBookingForUser[groupName] = userBookingForUserForGroup
print userBookingForUserForGroup
userBooking[userName] = userBookingForUser
print userBookingForUser
print userBooking
cHelper.setUserBookingTable(userBooking)
print "booking succeded"
return True
def __init__(self,arg):
self.userName = arg[0]
sConf = staticConfig()
self.logger = autoscaleLog(__file__)
self.ipInfoC = sConf.getInfoCLocation()["ipInfoC"]
self.portInfoC = sConf.getInfoCLocation()["portInfoC"]
self.cHelper = configHelper(self.ipInfoC,self.portInfoC)
hostName = self.executeCmd("hostname")
iframe = sConf.getHostInterface(hostName)
self.hostIP = self.getLocalIP(iframe)
def __init__(self):
self.logger = autoscaleLog(__file__)
sConf = staticConfig()
self.ipInfoC = sConf.getInfoCLocation()["ipInfoC"]
self.portInfoC = sConf.getInfoCLocation()["portInfoC"]
self.cHelper = configHelper(self.ipInfoC,self.portInfoC)
hostName = self.executeCmd("hostname")
iframe = sConf.getHostInterface(hostName)
self.hostIP = self.getLocalIP(iframe)
self.loadConf()
self.iscsiTargetType = sConf.getTargetType("groupManager")
for cmd in self.initialCmds:
self.executeCmd(cmd)
def run(arg):
logger = autoscaleLog(__file__)
sConf = staticConfig()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper(infoCLocation["ipInfoC"],infoCLocation["portInfoC"])
consumersConf = cHelper.getConsumerConf()
providersConf = cHelper.getProviderConf()
tagsManager = cHelper.getTagsManager()
allConf = {}
allConf["dev"] = providersConf
allConf["appserver"] = consumersConf
allConf["tags"] = tagsManager
logger.writeLog(allConf)
print json.dumps(allConf)
logger.shutdownLog()
return allConf
def run(arg):
cswitcher = codecSwitcher()
logger = autoscaleLog(__file__)
logger.writeLog(sys.argv)
groupName = cswitcher.getEncode(arg[0])
#tagList = []
#for a in arg[1:]:
# tagList.append(unicode(a))
tagList = cswitcher.getEncode(arg[1:])
sConf = staticConfig()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper( infoCLocation.get("ipInfoC"), infoCLocation.get("portInfoC"))
providersConf = cHelper.getProviderConf()
if providersConf.get(groupName) == None:
# providersConf[groupName] = {"tags":tagList}
providersConf[groupName] = {}
else:
print "group exist"
print "failed1failed2failed"
sys.exit(1)
logger.writeLog(providersConf)
print json.dumps(providersConf)
gmConf = cHelper.getGroupMConf()
groupAmount = len(gmConf)
if gmConf.get(groupName) == None:
gmConf[groupName] = {}
gmConf[groupName]["currentTid"] = 500+200*groupAmount
gmConf[groupName]["gmIP"] = sConf.getGroupMIP()
gmConf[groupName]["devicesLoaded"] = []
gmConf[groupName]["consumersLoaded"] = []
else:
print "group exist"
print "failed1failed2failed"
sys.exit(1)
logger.writeLog(gmConf)
print json.dumps(gmConf)
tagsManager = cHelper.getTagsManager()
if tagsManager.get(groupName) == None:
tagsManager[groupName] = tagList
else:
print "group exist"
print "failed1failed2failed"
sys.exit(1)
cHelper.setProviderConf(providersConf)
cHelper.setGroupMConf(gmConf)
cHelper.setTagsManager(tagsManager)
logger.shutdownLog()
def __init__(self, groupName):
sConf = staticConfig()
self.ipInfoC = sConf.getInfoCLocation()["ipInfoC"]
self.portInfoC = sConf.getInfoCLocation()["portInfoC"]
self.cHelper = configHelper(self.ipInfoC,self.portInfoC)
hostName = self.executeCmd("hostname")
iframe = sConf.getHostInterface(hostName)
self.path = sConf.getPath()
self.hostIP = self.getLocalIP(iframe)
self.groupName = groupName
cswitcher = codecSwitcher()
self.groupNameHash = cswitcher.getHash(groupName)
self.vgName = self.groupNameHash + "VG"
self.loadConf()
for cmd in self.initialCmds:
self.executeCmd(cmd)
self.logger = autoscaleLog(__file__)
def run(arg):
username = arg[0]
logger = autoscaleLog(__file__)
sConf = staticConfig()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper(infoCLocation["ipInfoC"], infoCLocation["portInfoC"])
consumersConf = cHelper.getConsumerConf()
userConsumers = cHelper.getUserConsumers()
consumers = userConsumers.get(username)
if consumers == None:
consumers = []
userConsumersConf = {}
for consumer in consumers:
conf = consumersConf.get(consumer)
if conf != None:
userConsumersConf[consumer] = conf
logger.writeLog(userConsumersConf)
print json.dumps(userConsumersConf)
logger.shutdownLog()
def __init__(self,deviceName,groupName):
sConf = staticConfig()
self.ipInfoC = sConf.getInfoCLocation()["ipInfoC"]
self.iscsiTargetType = sConf.getTargetType("storageProvider")
self.portInfoC = sConf.getInfoCLocation()["portInfoC"]
self.cHelper = configHelper(self.ipInfoC,self.portInfoC)
hostName = self.executeCmd("hostname")
iframe = sConf.getHostInterface(hostName)
self.hostIP = self.getLocalIP(iframe)
self.conf["deviceName"] = deviceName
self.conf["deviceGroup"] = groupName
self.loadConf()
for cmd in self.initialCmds:
self.executeCmd(cmd)
self.logger = autoscaleLog(__file__)
sConf = staticConfig()
self.path = sConf.getPath()+"core/"
confPath = sConf.getPath()+"conf/storageProviderConf.json"
with open(confPath,"r") as fspc:
self.storageProviderConf = json.loads(fspc.read())
def run(arg):
cswitcher = codecSwitcher()
sConf = staticConfig()
path = sConf.getPath()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper(infoCLocation["ipInfoC"],infoCLocation["portInfoC"])
groupName = cswitcher.getEncode(arg[0])
gmsConf = cHelper.getGroupMConf()
gmConf = gmsConf.get(groupName)
if gmConf == None:
print "group do not exist"
print "failed1failed2failed"
return False
gmIP = gmConf.get("gmIP")
print gmIP
print path
print groupName
deleteGroupCmd = u"ssh -t root@"+gmIP+u" \"python "+path+u"main.py deleteGroup "+groupName+u"\""
out = executeCmd(deleteGroupCmd)
if out.find("706errorKEY") >= 0:
print "failed1failed2failed"
sys.exit(1)
def __init__(self):
self.logger = autoscaleLog(__file__)
self.sConf = staticConfig()
infoCLocation = self.sConf.getInfoCLocation()
self.cHelper = configHelper( infoCLocation.get("ipInfoC"), infoCLocation.get("portInfoC"))
def run(arg):
cswitcher = codecSwitcher()
logger = autoscaleLog(__file__)
logger.writeLog(arg)
sConf = staticConfig()
path = sConf.getPath()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper( infoCLocation.get("ipInfoC"), infoCLocation.get("portInfoC"))
print arg
userName = arg[0]
consumerLocation = arg[1]
stepSize = int(arg[2])
tagList = cswitcher.getEncode(arg[3:])
gchooser = groupChooser()
groupList = gchooser.chooseGroup(tagList)
if groupList == []:
print "No storage resource available"
print "failed1failed2failed"
return False
groupName = groupList[0]
# currentTime = time.time()
timeM = timeManager()
currentTime = timeM.getTime()
# print "currentTime"
# print currentTime
ctKey = str(currentTime/3600)
# print "ctKey"
# print ctKey
userBooking = cHelper.getUserBookingTable()
# print "userBooking GET"
# print userBooking
userBookingForUser = userBooking.get(userName)
# print "userBookingForUser"
# print userBookingForUser
if userBookingForUser == None:
print "User did not book any storage"
print "failed1failed2failed"
return False
userBookingForUserForGroup = userBookingForUser.get(groupName)
# print "userBookingForUserForGroup"
# print userBookingForUserForGroup
if userBookingForUserForGroup == None:
print "User did not book storage for "+str(tagList)
print "failed1failed2failed"
return False
bookedStorageSize = userBookingForUserForGroup.get(ctKey)
# print "bookedStorageSize"
# print bookedStorageSize
if bookedStorageSize == None:
print "User did not book storage for this period"
print "failed1failed2failed"
return False
bookedStorageSize = int(bookedStorageSize)
if bookedStorageSize < stepSize:
print "User do not have enough booked storage space"
print "failed1failed2failed"
return False
# check current available space, if not enough , release
currentUsageInfo = getUsageInfo.run([])
groupSize = currentUsageInfo.get(groupName).get('groupSize')
usedSize = currentUsageInfo.get(groupName).get('usedSize')
asize = groupSize - usedSize
neededSize = 0
if stepSize > asize:
neededSize = stepSize - asize
if neededSize > 0:
releaseConsumerList = []
releaseCandidates = cHelper.getReleaseCandidates()
for deviceMapConsumerIP,size in releaseCandidates.items():
neededSize = neededSize - size
releaseConsumerList.append(deviceMapConsumerIP)
if neededSize <= 0:
break
if neededSize > 0:
print "Do not have enough space sorry"
return False
# release consumers
for deviceMapConsumerIP in releaseConsumerList:
deviceMap, consumerIP = deviceMapConsumerIP.split("@")
releaseExtraStorage.run([consumerIP,deviceMap])
requestStorageCmd = "ssh -t root@"+consumerLocation+" \"python "+path+"main.py requestStorage "+groupName+" "+str(stepSize)+" booked\""
print requestStorageCmd
out = executeCmd(requestStorageCmd)
if out.find("706errorKEY") >= 0:
print "failed1failed2failed"
sys.exit(1)
# userBookingForUserForGroup[ctKey] = bookedStorageSize - stepSize
startKeyInt = int(ctKey)
timeKeys = userBookingForUserForGroup.keys()
timeKeysInt = []
for timeKey in timeKeys:
timeKeysInt.append(int(timeKey))
timeKeysInt.sort()
for i in xrange(len(timeKeysInt)):
timeKeyInt = timeKeysInt[i]
if timeKeyInt < startKeyInt:
userBookingForUserForGroup.pop(cswitcher.getEncode(timeKeyInt))
elif timeKeyInt == startKeyInt:
timeKey = cswitcher.getEncode(timeKeyInt)
timeKeyBookedSize = userBookingForUserForGroup[timeKey]
userBookingForUserForGroup[timeKey] = timeKeyBookedSize - stepSize
startKeyInt += 1
else:
break
cHelper.setUserBookingTable(userBooking)
print "get booked storage succeded"
return True
def run(arg):
logger = autoscaleLog(__file__)
sConf = staticConfig()
infoCLocation = sConf.getInfoCLocation()
cHelper = configHelper(infoCLocation["ipInfoC"], infoCLocation["portInfoC"])
usersBooking = cHelper.getUserBookingTable()
print usersBooking
tagManager = cHelper.getTagsManager()
userName = unicode(arg[0], "utf-8")
print userName
userBooking = usersBooking.get(userName)
print userBooking
userBookingRecords = []
if userBooking == None:
print json.dumps(userBookingRecords)
return userBookingRecords
for group, bookingTable in userBooking.items():
tags = tagManager.get(group)
userBookingForGroup = userBooking.get(group)
timestamps = []
for t in userBookingForGroup.keys():
timestamps.append(int(t))
timestamps.sort()
print timestamps
try:
bookingRecord = {}
startT = timestamps[0]
startS = userBookingForGroup.get(str(startT))
endT = None
for i in xrange(len(timestamps)):
middleT = timestamps[i]
endT = middleT
middleS = userBookingForGroup.get(str(middleT))
print middleT, middleS, timestamps[i + 1]
print endT
print middleT != (timestamps[i + 1] - 1)
print middleS != userBookingForGroup.get(str(timestamps[i + 1]))
print (middleT != (timestamps[i + 1] - 1)) or (
middleS != userBookingForGroup.get(str(timestamps[i + 1]))
)
if (middleT != (timestamps[i + 1] - 1)) or (middleS != userBookingForGroup.get(str(timestamps[i + 1]))):
bookingRecord["StartTime"] = time.ctime(startT * 3600)
bookingRecord["EndTime"] = time.ctime(endT * 3600 + 3600)
bookingRecord["Size"] = "%s MB" % (str(startS))
bookingRecord["User"] = userName
bookingRecord["Tag1"] = tags[0]
bookingRecord["Tag2"] = tags[1]
userBookingRecords.append(bookingRecord)
bookingRecord = {}
startT = timestamps[i + 1]
startS = userBookingForGroup.get(str(startT))
except:
bookingRecord["StartTime"] = time.ctime(startT * 3600)
bookingRecord["EndTime"] = time.ctime(endT * 3600 + 3600)
bookingRecord["Size"] = "%s MB" % (str(startS))
bookingRecord["User"] = userName
bookingRecord["Tag1"] = tags[0]
bookingRecord["Tag2"] = tags[1]
userBookingRecords.append(bookingRecord)
print json.dumps(userBookingRecords)
return userBookingRecords
print json.dumps(userBookingRecords)
return userBookingRecords