def getServersInfo(serverNameInfo,clusterName):
sum = 0
online = 0
serversInfo = []
servers = []
sum = len(serverNameInfo)
totalSpace = Decimal("0.0")
usedSpace = Decimal("0.0")
strs = str(clusterName) + "_servers"
message = Globals.mc.get(strs)
if message is not None:
serversInfo = message
else:
for server in serverNameInfo:
serverName = str(server.name)
servers.append(serverName)
if Utils.isOnline(serverName) == True:
cmd = 'python ' + Globals.BACKEND_SCRIPT + 'get_server_bricks.py'
status,message = Utils.executeOnServer(serverName, cmd)
if status == 0:
if message.strip() != '[]':
message = message.replace("['",'')
message = message.replace("']",'')
info = message.split(':')
usedSpace = usedSpace + Decimal(info[2])
totalSpace = totalSpace + Decimal(info[3])
online = online + 1
serversInfo.append(online)
serversInfo.append(sum)
serversInfo.append(usedSpace)
serversInfo.append(totalSpace)
serversInfo.append(servers)
return serversInfo
def getNeofsStatus(clusterName, serverName):
code = None
reval = None
params = []
params.append(clusterName)
params.append(serverName)
status, clusterid, server_id = get_server(clusterName, serverName)
if status == "-1":
code, reval = "20052", "No cluster " + clusterName + "."
elif status == "-2":
code, reval = "20054", "server " + serverName + " is not in cluster " + clusterName + "."
elif Utils.isOnline(serverName) == False:
code, reval = "22008", "Server " + serverName + " is not online."
if code is not None:
result = Utils.errorCode(code, reval, params)
logger.error(reval)
raise web.HTTPError(status="400 Bad Request", data=result)
try:
commandWithArgs = "sh " + Globals.BACKEND_SCRIPT + "neofs_status.sh"
status, message = Utils.executeOnServer(serverName, commandWithArgs)
responseDom = XmlHandler.ResponseXml()
tasksTag = responseDom.appendTagRoute("responseStatus")
tasksTag.appendChild(responseDom.createTag("status", message))
logger.info("Get neofs mount status: " + message)
return tasksTag.toxml()
except Exception, e:
code, reval = "22202", "Error when get neofs status on server:" + str(e)
def removeServerFromCluster(clusterName, serverName):
code = None
reval = None
params = []
params.append(clusterName)
params.append(serverName)
status, clusterid, server_id = get_server(clusterName, serverName)
if status == "-1":
code, reval = "20052", "No cluster " + clusterName + "."
elif status == "-2":
code, reval = "20054", "server " + serverName + " is not in cluster " + clusterName
elif Utils.isOnline(serverName) == False:
code, reval = "22008", "Server " + serverName + " is not online"
if code is not None:
result = Utils.errorCode(code, reval, params)
logger.error(reval)
raise web.HTTPError(status="400 Bad Request", data=result)
try:
servers = Globals.db.select("server_info", what="*", where="cluster_id=$clusterid", vars=locals())
if len(servers) == 0:
logger.info("This server[" + serverName + "] not contained in table server_info")
return ""
if len(servers) == 1:
Globals.db.delete("server_info", where="id=$server_id", vars=locals())
logger.info("delete server[" + serverName + "] from table server_info")
return ""
for server in servers:
if (Utils.getIPByName(server.name) != Utils.getIPByName(serverName)) and (
server.name.strip() != serverName
):
break
if Utils.isLocalHost(serverName):
status, ip = Utils.getLocal_IP()
else:
ip = Utils.getIPByName(serverName)
status, msg = Utils.executeOnServer(server.name, "gluster --mode=script peer detach " + ip)
if status == -1:
code, reval = (
"26104",
"error when connecting to remote host " + serverName + " from localhost " + server.name + "." + msg,
)
elif status == -2:
code, reval = "26059", "Error when using pub key to connect remote server " + serverName + "." + msg
elif status == 1:
code, reval = "22014", 'Error when executing "gluster --mode=script peer detach ' + ip + '"' + msg
if code is not None:
result = Utils.errorCode(code, reval, params)
web.HTTPError(status="400 Bad Request", data="")
logger.error(reval)
return result
Globals.db.delete("server_info", where="id=$server_id", vars=locals())
return ""
except Exception, e:
code, reval = "22101", "failed to remove server " + serverName + "." + str(e)
logger.error(reval)
result = Utils.errorCode(code, reval, params)
raise web.HTTPError(status="400 Bad Request", data=result)
def operationCluster(clusterName):
try:
cluster_id_info = db.select('cluster_info',where='name=$clusterName' ,what="id",vars=locals())
if len(cluster_id_info) == 0:
code, reval = "20200", "Error ["+clusterName+"] not existed"
else:
data = web.input()
operation = data.operation
status,local_IP = Utils.getLocal_IP()
localIP = local_IP.strip()
cluster_id = cluster_id_info[0].id
server_name_info = db.select('server_info',where='cluster_id=$cluster_id' ,what="name",vars=locals())
if operation == "mount":
for server in server_name_info:
serverName = server.name
commandWithArgs = "sh " + Globals.BACKEND_SCRIPT + "neofs_setup.sh " + serverName
status,message = Utils.executeOnServer(serverName,commandWithArgs)
if status != 0:
code, reval = "21015", "Error when operation[mount] cluster[" + clusterName + "]: mount failure on server[" + serverName + "]"
break
if status == 0:
logger.info("Operation[mount] cluster[" + clusterName + "] successfully!")
return ''
elif operation == "umount":
for server in server_name_info:
serverName = server.name
commandWithArgs = "sh " + Globals.BACKEND_SCRIPT + "neofs_teardown.sh " + serverName
status,message = Utils.executeOnServer(serverName,commandWithArgs)
if status != 0:
code, reval = "21015", "Error when operation[umount] cluster[" + clusterName + "]: umount failure on server[" + serverName + "]"
break
if status == 0:
logger.info("Operation[umount] cluster[" + clusterName + "] successfully!")
return ''
elif operation == "remount":
for server in server_name_info:
serverName = server.name
commandWithArgs = "sh " + Globals.BACKEND_SCRIPT + "neofs_teardown.sh " + serverName
status,message = Utils.executeOnServer(serverName,commandWithArgs)
if status != 0:
code, reval = "21015", "Error when operation[remount] cluster[" + clusterName + "]: umount failure on server[" + serverName + "]"
break
commandWithArgs = "sh " + Globals.BACKEND_SCRIPT + "neofs_setup.sh " + serverName
status,message = Utils.executeOnServer(serverName,commandWithArgs)
if status != 0:
code, reval = "21015", "Error when operation[remount] cluster[" + clusterName + "]: mount failure on server[" + serverName + "]"
break
if status == 0:
logger.info("Operation[remount] cluster[" + clusterName + "] successfully!")
return ''
else:
code, reval = "21014", "Error when operation cluster[" + clusterName + "]: no such operation on cluster"
except Exception, e:
code, reval = "21015", "Error when operation a cluster:" + str(e)
def get_disk_info(self, reference):
server_disk = reference.split(':')
chkcmd = 'python '+ Globals.BACKEND_SCRIPT + '/get_format_device_status.py ' + server_disk[1]
hostName = os.popen('hostname').read()
if Utils.isLocalHost(server_disk[1].strip()):
(status, output) = commands.getstatusoutput(chkcmd)
else:
(status, output) = Utils.executeOnServer(server_disk[0].strip(), chkcmd)
return status,output
def getDiskStatus(reference):
server_disk = reference.split(":")
chkcmd = "python " + Globals.BACKEND_SCRIPT + "get_format_device_status.py " + server_disk[1]
if Utils.isLocalHost(server_disk[0].strip()):
(status, output) = commands.getstatusoutput(chkcmd)
else:
(status, output) = Utils.executeOnServer(server_disk[0].strip(), chkcmd)
return (status, output)
def setUid(uid, userName):
try:
fp = open(Globals.CIFS_USER_FILE, "a")
fp.write("%s:%s\n" % (uid, userName))
fp.close()
return True
except IOError, e:
Utils.log("failed to write file %s: %s" % (Globals.CIFS_USER_FILE, str(e)))
return False
def removeUser(userName):
lines = Utils.readFile(Globals.CIFS_USER_FILE, lines=True)
try:
fp = open(Globals.CIFS_USER_FILE, "w")
for line in lines:
if not line.strip():
continue
## junli.li remove the line of userName from CIFS_USER_FILE
if line.strip().split(":")[1] == userName:
continue
fp.write("%s" % line)
fp.close()
except IOError, e:
Utils.log("failed to write file %s: %s" % (Globals.CIFS_USER_FILE, str(e)))
return False
def deleteUser(data):
try:
userName = data.userName
except Exception,e:
code, reval = "25106", "Error when deleting user: "******"400 Bad Request", data = result)
def getUid(userName):
lines = Utils.readFile(Globals.CIFS_USER_FILE, lines=True)
for line in lines:
tokens = line.strip().split(":")
if tokens[1] == userName:
return int(tokens[0])
return None
def getCifsUsers():
try:
lines = Utils.readFile(Globals.CIFS_USER_FILE, lines=True)
cifsUserDom = XmlHandler.ResponseXml()
cifsUserTag = cifsUserDom.appendTagRoute("cifsUsers")
for line in lines:
if not line.strip():
continue
tokens = line.strip().split(":")
cifsUserTag.appendChild(cifsUserDom.createTag("cifsUser", tokens[1]))
return cifsUserTag.toxml()
except Exception,e:
code, reval = "25000", "Error when getting cifsusers: " + str(e)
result = Utils.errorCode(code, reval, [])
raise web.HTTPError(status = "400 Bad Request", data = result)
def getOnlineServer(cluster_id):
server_name_info = db.select('server_info',where='cluster_id=$cluster_id' ,what="name",vars=locals())
servers = []
for server in server_name_info:
if Utils.isOnline(server.name) == True:
serverName = server.name
servers.append(serverName.strip())
return servers
def addUser(data):
try:
userName = data.userName
password = data.password
except Exception,e:
code, reval = "25102", "Error when adding user: "******"400 Bad Request", data = result)
def initDisk(clusterName, serverName, diskName, data):
try:
fsType = data.fsType
mountPoint = data.mountPoint
except Exception, e:
code, reval = "22011", "Error when getting servers list:fsType and mountPoint are required." + str(e)
logger.error(reval)
result = Utils.errorCode(code, reval, [])
raise web.HTTPError(status="400 Bad Request", data=result)
def changeUserPassword(data):
try:
userName = data.userName
newPassword = data.newPassword
oldPassword = data.oldPassword
except Exception,e:
code, reval = "25103", "Error when changing user's password: "******"400 Bad Request", data = result)
def deleteCifsUser(data):
code = "25003"
reval = ""
try:
userName = data.userName
except Exception,e:
reval = "Error when deleting cifs user: "******"400 Bad Request", data = result)
def addServerToCluster(clusterName, data):
code = None
reval = None
try:
serverName = data.serverName
except Exception, e:
code, reval = "22010", "Error when getting servers list:serverName is required" + str(e)
logger.error(reval)
result = Utils.errorCode(code, reval, [])
raise web.HTTPError(status="400 Bad Request", data=result)
def changeCifsUserPasswd(data):
code = "25004"
reval = ""
try:
userName = data.userName
password = data.password
except Exception,e:
reval = "Error when changing cifs user: "******"400 Bad Request", data = result)
def getServer(clusterName, serverName):
status, cluster_info, server_info = get_server(clusterName, serverName)
code = None
reval = None
if status == "-1":
code, reval = "20052", "No cluster " + clusterName
elif status == "-2":
code, reval = "20054", "No server " + serverName + " in cluster " + clusterName
elif Utils.isOnline(serverName) == False:
code, reval = "22008", "Server " + serverName + " is not online"
if code is not None:
result = Utils.errorCode(code, reval, [])
logger.error(reval)
raise web.HTTPError(status="400 Bad Request", data=result)
try:
serverName = str(serverName)
message = Globals.mc.get(serverName + "_server")
if message is not None:
logger.info("Get server[" + serverName + "]'s details from memcache:" + message)
return message
cmd = "python " + Globals.BACKEND_SCRIPT + "get_server_details.py"
status, message = Utils.executeOnServer(serverName, cmd)
if status == -1:
code, reval = "26104", message
elif status == -2:
code, reval = "26059", "Error when using pub key to connect remote server " + serverName + "." + message
elif status == 1:
code, reval = "22002", "error when getting details from server " + serverName + "." + message
if code is not None:
result = Utils.errorCode(code, reval, [])
logger.error(reval)
raise web.HTTPError(status="400 Bad Request", data=result)
logger.info("Get server[" + serverName + "]'s details by executing get_server_details.py:" + message)
return message
except Exception, e:
code, reval = "22002", "Error when getting servers list:" + str(e)
result = Utils.errorCode(code, reval, [])
logger.error(reval)
raise web.HTTPError(status="400 Bad Request", data=result)
def getVolumesInfo(clusterName):
strs = str(clusterName) + "_volumes"
message = Globals.mc.get(strs)
if message is not None:
return message
onlineNum = 0
totalNum = 0
volumes = []
cluster_info = Globals.db.select('cluster_info',where='name=$clusterName',what='*',vars=locals())
clusterid = cluster_info[0].id
server_info = Globals.db.select('server_info', where='cluster_id=$clusterid',vars=locals())
for server in server_info:
if Utils.isOnline(str(server.name)) == False:
continue
status, message = Utils.executeOnServer(str(server.name), 'gluster volume list')
if message.strip() != 'No volumes present in cluster':
volumeList = message.split('\n')
if len(volumeList) != 0:
onlineNum = 0
totalNum = 0
volumes = []
for v in volumeList:
if v.strip() != '':
cmd = "python " + Globals.BACKEND_SCRIPT + "get_volumes.py " + v
status,message = Utils.executeOnServer(str(server.name), cmd)
totalNum += 1
volumes.append(v)
if status == 0:
weblog = xml.etree.ElementTree.fromstring(message)
for entry in weblog.findall('status'):
status = entry.text
if status.strip().lower() == 'online':
onlineNum += 1
clusterVolumeInfo = []
clusterVolumeInfo.append(onlineNum)
clusterVolumeInfo.append(totalNum)
clusterVolumeInfo.append(volumes)
return clusterVolumeInfo
def getUsers():
try:
responseDom = XmlHandler.ResponseXml()
usersTag = responseDom.appendTagRoute("users")
userNameList = db.select('users',what="username")
if len(userNameList) == 0:
return '<users></users>'
for user in userNameList:
usersTag.appendChild(responseDom.createTag("user", user.username))
return usersTag.toxml()
except Exception, e:
code, reval = "25101", "Error when getting users list:" + str(e)
result = Utils.errorCode(code, reval, [])
raise web.HTTPError(status = "400 Bad Request", data = result)
def migrateBrick(clusterName, volumeName):
autoCommit = False
try:
data = web.input()
serverName, code, reval = isPathExist(clusterName, volumeName, data)
if code == 0:
source = data.source
target = data.target
server_dir = target.split(":")
target = Utils.getIPByName(server_dir[0].strip()) + ":" + server_dir[1]
commandWithArgs = "gluster volume replace-brick " + volumeName + " " + source + " " + target + " start"
status, message = Utils.executeOnServer(serverName, commandWithArgs)
if status != 0:
code, reval = "23012", "Error when migrate a brick:" + message
else:
description = "Brick Migration on volume [" + volumeName + "] from [" + source + "] to [" + target + "]"
reference = volumeName + "#" + source + "#" + target
operation_id = 2
db.insert(
"task_info",
description=description,
reference=reference,
operation_id=operation_id,
cluster_name=clusterName,
)
autoCommit_str = data.autoCommit
if autoCommit_str is not None:
if autoCommit_str == "true":
autoCommit = True
else:
autoCommit = False
logger.info(description)
return ""
except Exception, e:
if str(e) == "'autoCommit'":
return ""
code, reval = "23012", "Error when migrate a brick:" + str(e)
def addBrick(clusterName, volumeName):
try:
data = web.input()
serverName, code, reval = isPathExist(clusterName, volumeName, data)
if code == 0:
bricks = data.bricks
bricks = bricks.replace(" ", ",")
brickList = bricks.split(",")
brickList = VolumeUtils.convertToIPAddr(brickList)
bricks = ""
for brick in brickList:
bricks += brick + " "
commandWithArgs = "gluster volume add-brick " + volumeName + " " + bricks
status, message = Utils.executeOnServer(serverName, commandWithArgs)
if status == 0:
logger.info("Add bricks to volume[" + volumeName + "]")
return ""
else:
code, reval = "23011", "Error when add a brick: " + message
except Exception, e:
code, reval = "23011", "Error when add a brick: " + str(e)
def operationServer(clusterName, serverName):
code = None
reval = None
params = []
params.append(clusterName)
params.append(serverName)
status, clusterid, server_id = get_server(clusterName, serverName)
if status == "-1":
code, reval = "20052", "No cluster " + clusterName + "."
elif status == "-2":
code, reval = "20054", "server " + serverName + " is not in cluster " + clusterName
elif Utils.isOnline(serverName) == False:
code, reval = "22008", "Server " + serverName + " is not online"
if code is not None:
result = Utils.errorCode(code, reval, params)
logger.error(reval)
raise web.HTTPError(status="400 Bad Request", data=result)
try:
data = web.input()
operationName = data.operation
if operationName == "mount":
commandWithArgs = "sh " + Globals.BACKEND_SCRIPT + "neofs_setup.sh " + serverName
status, message = Utils.executeOnServer(serverName, commandWithArgs)
return ""
elif operationName == "umount":
commandWithArgs = "sh " + Globals.BACKEND_SCRIPT + "neofs_teardown.sh " + serverName
status, message = Utils.executeOnServer(serverName, commandWithArgs)
return ""
elif operationName == "remount":
commandWithArgs = "sh " + Globals.BACKEND_SCRIPT + "neofs_teardown.sh " + serverName
status, message = Utils.executeOnServer(serverName, commandWithArgs)
commandWithArgs = "sh " + Globals.BACKEND_SCRIPT + "neofs_setup.sh " + serverName
status, message = Utils.executeOnServer(serverName, commandWithArgs)
return ""
else:
return "22200", "Error no such operation on server"
except Exception, e:
code, reval = "22201", "Error when operation a cluster:" + str(e)
def errorreturn(code,reval,params):
result = Utils.errorCode(code, reval, params)
raise web.HTTPError(status = "400 Bad Request", data = result)
def removeBrick(clusterName, volumeName):
try:
data = web.input()
serverName, code, reval = isPathExist(clusterName, volumeName, data)
dataLen = len(data)
if code == 0:
if dataLen == 1:
bricks = data.bricks
bricks = bricks.replace(",", " ")
commandWithArgs = (
"gluster --mode=script volume remove-brick " + volumeName + " " + bricks + " " + " force"
)
status, message = Utils.executeOnServer(serverName, commandWithArgs)
if status != 0:
code, reval = "23013", "Error where delete a brick" + message
else:
return ""
elif dataLen == 2:
deleteData = data.deleteData
bricksInfo = data.bricks
bricks = bricksInfo.replace(",", " ")
commandWithArgs = (
"gluster --mode=script volume remove-brick " + volumeName + " " + bricks + " " + " force"
)
status, message = Utils.executeOnServer(serverName, commandWithArgs)
if status != 0:
code, reval = "23013", "Error where delete a brick: " + message
else:
if deleteData == "true":
bricks = bricksInfo.split(",")
for brick in bricks:
brickSplit = brick.split(":")
brickServer = brickSplit[0]
brickDir = brickSplit[1]
commandWithArgs = "rm -rf " + brickDir
status, message = Utils.executeOnServer(brickServer, commandWithArgs)
if status != 0:
code, reval = "23013", "Error where delete a brick: " + message
break
if status == 0:
return ""
else:
return ""
elif dataLen == 3:
deleteData = data.deleteData
bricksInfo = data.bricks
bricks = bricksInfo.replace(",", " ")
commandWithArgs = (
"gluster --mode=script volume remove-brick " + volumeName + " " + bricks + " " + " force"
)
status, message = Utils.executeOnServer(serverName, commandWithArgs)
if status != 0:
code, reval = "23013", "Error where delete a brick: " + message
else:
if deleteData == "true":
bricks = bricksInfo.split(",")
for brick in bricks:
brickSplit = brick.split(":")
brickServer = brickSplit[0]
brickDir = brickSplit[1]
commandWithArgs = "rm -rf " + brickDir
status, message = Utils.executeOnServer(brickServer, commandWithArgs)
if status != 0:
code, reval = "23013", "Error where delete a brick: " + message
break
if status == 0:
return ""
else:
return ""
else:
code, reval = "30002", "Input data is too long "
except Exception, e:
code, reval = "23013", "Error where delete a brick: " + str(e)
def isVolumeExist(serverName, volumeName):
commandWithArgs = "gluster volume info " + volumeName
status, message = Utils.executeOnServer(serverName, commandWithArgs)
if status == 0:
return True
return False
else:
rebal = Rebalance_operation()
(code, reval) = rebal.get_rebalance_info(task.reference)
messages = reval.split('\n')
message = messages[len(messages) - 1].strip()
code, message = getRebalanceStatus(reval)
percentCompleted = '0.0'
task_info_xml = addTaskTag(tasksTag,task.taskID,task.commitSupported,task.description,task.taskID,task.pauseSupported,task.reference,code,message,percentCompleted,task.percentageSupported,task.stopSupported,task.operation_type)
logger.info("Get all tasks info")
return task_info_xml
except Exception,e:
code, reval = "24001", "Error when get all tasks info:" + str(e)
logger.error(reval)
params = []
params.append(clusterName)
result = Utils.errorCode(code, reval, params)
raise web.HTTPError(status = "400 Bad Request", data = result)
def getTask(clusterName,taskID):
try:
cluster_id_info = db.select('cluster_info',where='name=$clusterName',what="id",vars=locals())
if len(cluster_id_info) == 0:
code, reval = "20052", "Error cluster: "+clusterName+" not exist"
else:
task_info = db.select('task_info,operation_info',where='task_info.operation_id=operation_info.id and task_info.id=$taskID and task_info.cluster_name=$clusterName',what = "*",vars=locals())
if len(task_info) == 0:
code, reval = "20056", "Error the task not exist"
else:
task =task_info[0]
if task.operation_id == 2:
task_info=task.reference.replace('#',' ')
password = data.password
except Exception,e:
code, reval = "25102", "Error when adding user: "******"400 Bad Request", data = result)
try:
db.select('users',where='username=$userName' ,what="*",vars=locals())
password = hashlib.sha1("sAlT754-"+password).hexdigest()
db.insert('users',username=userName, password=password, enabled=1)
return None
except Exception, e:
code, reval = "25102", "Error when adding user" + userName + ":" + str(e)
params = []
params.append(userName)
result = Utils.errorCode(code, reval, params)
raise web.HTTPError(status = "400 Bad Request", data = result)
def changeUserPassword(data):
try:
userName = data.userName
newPassword = data.newPassword
oldPassword = data.oldPassword
except Exception,e:
code, reval = "25103", "Error when changing user's password: "******"400 Bad Request", data = result)
try:
password_info = db.select('users',where='username=$userName' ,what="password",vars=locals())
password_hash = password_info[0].password
def getLastUid():
lines = Utils.readFile(Globals.CIFS_USER_FILE, lines=True)
if not lines:
return Globals.DEFAULT_UID
return int([line.strip().split(':')[0] for line in lines if line.strip()][-1])
