def createVolume(clusterName,data):
serverName = getServerName(clusterName)
cmd, options, volumeName = preCreateVolume(data)
volume_info = Globals.db.select('volume_info', what='*', where='name=$volumeName', vars=locals())
if len(volume_info) > 0:
params = []
params.append(volumeName)
result = Utils.errorCode('23015','error while adding a volume:volume ' + volumeName + ' already exists', params)
logger.error('error while adding a volume:volume ' + volumeName + ' already exists')
raise web.HTTPError(status = "400 Bad Request", data = result)
#################################execute on localhost or remote host
status,output = Utils.executeOnServer(serverName, cmd)
if status == -1:
code, reval = '26104', 'error when connecting to remote host ' + serverName + ' from localhost:' + output
elif status == -2:
code,reval = '26059', 'Error when using pub key to connect remote server ' + serverName + '.' + output
elif status == 1:
if re.match('exists', output):
code, reval = '20103', output
else:
code, reval = '23015', 'Error when creating volume' + volumeName + '.' + output
if status:
result = Utils.errorCode(code, reval, [])
logger.error(reval)
raise web.HTTPError(status = "400 Bad Request", data = result)
optionToken = options.split(",")
hostAllowed = "\*"
for option in optionToken:
if option.split("=")[0] == "auth.allow":
hostAllowed = option.split("=")[1]
nasProtocols = data.accessProtocols
if "CIFS" in nasProtocols:
isShared = data.isShared
owner = data.owner
if not createVolumeCIFSUsers(volumeName, isShared, owner, hostAllowed):
params = []
params.append(volumeName)
result = Utils.errorCode('23201', 'failed to configure cifs for volume: %s' % volumeName, params)
Utils.executeOnServer(serverName, 'gluster --mode=script volume delete ' + volumeName)
logger.error("failed to configure cifs for volume:" + volumeName)
raise web.HTTPError(status = '400 Bad Request',data = result)
try:
cluster = Globals.db.select('cluster_info',where='name=$clusterName',vars=locals())
clusterid=cluster[0].id
Globals.db.insert('volume_info',name=volumeName,cluster_id=clusterid)
except:
params = []
params.append(volumeName)
result = Utils.errorCode('23015','error while adding a volume:error while inserting volume_info into DB', params)
deleteVolumeCIFSUser(volumeName)
Utils.executeOnServer(serverName, 'gluster --mode=script volume delete ' + volumeName)
#postDelete(volumeName, brickList,deleteFlag)
logger.error('error while adding a volume:error while inserting volume_info into DB')
raise web.HTTPError(status = '400 Bad Request',data = result)
return None
def downloadLogs(clusterName, volumeName):
params = []
web.header('Content-Type','application/octet-stream')
serverName = getServerName(clusterName)
hostName = os.popen('hostname').read()
cmd = 'python ' + BACKEND_SCRIPT + 'get_volume_bricks.py ' + volumeName
(status,output) = Utils.executeOnServer(serverName, cmd)
list = output.split(' ')
if (list[-1] == 'exist\n') or (list[-1] == 'exist'):
params.append(volumeName)
result = Utils.errorCode('20053', 'volume ' + volumeName + ' does not exist', params)
raise web.HTTPError(status='400 Bad Request', data=result)
isExecSucess(status, output, serverName, volumeName, cmd, '23019',[])
bricklist = output.split('\n')
if status != 0:
return output
result = ''
if os.path.exists(TMP_LOG_DIR) == False:
os.mkdir(TMP_LOG_DIR)
logs = ''
for brick in bricklist:
server_dir = brick.split(':/')
if len(server_dir) != 2:
break
file = server_dir[1].strip()
log = file.replace('/', '-') + '.log'
status,files = Utils.executeOnServer(server_dir[0].strip(),'ls ' + BRICK_LOG_DIR +'|grep ^' + log)
files = files.split('\n')
for file in files:
if file.strip() is '':
break
cmd = 'scp ' + str(BRICK_LOG_DIR) + str(file) + \
' root@' + str(hostName).strip().split('\n')[0] + \
':' + TMP_LOG_DIR + '/' + str(server_dir[0].strip()) + '-' + file
(status,output) = Utils.executeOnServer(server_dir[0].strip(), cmd)
logs += ' ' + server_dir[0].strip() + '-' + file
if output[:7] == 'Warning':
continue
isExecSucess(status, output, serverName, volumeName, cmd, '23019', [])
if os.path.exists(TMP_LOG_DIR + volumeName + '.tar.gz'):
os.remove(TMP_LOG_DIR + volumeName + '.tar.gz')
os.chdir(TMP_LOG_DIR)
log = TMP_LOG_DIR + volumeName + '.tar.gz'
if os.path.exists(log):
os.remove(log)
os.popen('tar cvzf ' + volumeName + '.tar.gz ' + logs)
os.popen('rm -rf ' + logs)
try:
f=open(volumeName+'.tar.gz')
stream = f.read()
except Exception,e:
params.append(volumeName)
result = Utils.errorCode('23019', 'error when downloading logs', params)
raise web.HTTPError(status='500 Internal Error', data=result)
def preCreateVolume(data):
if len(data) == 0:
return 'clusterName is required'
code = None
count = None
try:
volumeName = data.volumeName
volumeType = data.volumeType
transportType = data.transportType
if data.replicaCount != '0':
count = data.replicaCount
if data.stripeCount != '0':
count = data.stripeCount
# # # accessProtocols = data.accessProtocols
options = data.options
bricks = data.bricks
nasProtocols = data.accessProtocols
if 'CIFS' in nasProtocols:
isShared = data.isShared
owner = data.owner
except Exception,e:
params = []
params.append(str(e))
result = Utils.errorCode('20001', 'paramters are required {0}', params)
raise web.HTTPError(status = '400 Bad Request', data = result)
def postDelete(volumeName, brickList,deleteFlag = False):
params = []
params.append(volumeName)
params.append(brickList)
params.append(deleteFlag)
for brick in brickList:
if brick.strip() is '':
continue
cmd = 'python ' + BACKEND_SCRIPT + 'clear_volume_directory.py'
server_dir = brick.split(":/")
if len(server_dir) != 2:
break
cmd += ' /' + server_dir[1].strip()
status,output = Utils.executeOnServer(server_dir[0].strip(), cmd)
if status == -1:
params = []
params.append(volumeName)
params.append(server_dir[0].strip())
params.append(output)
code, reval = '26104', 'Volume {0} deleted from cluster, however following error(s) occurred:\nerror when connecting to remote host {1} from localhost:{2}'
elif status == -2:
code,reval = '26059', 'Volume {0} deleted from cluster, however following error(s) occurred:\nError when using pub key to connect remote server {1}.{2}'
elif status == 1:
if re.match('exist', output) or re.match('exists', output) or re.match('exist\n',output):
code, reval = '20053', 'volume {0} does not exist.\n'
else:
code, reval = '23101', 'Volume {0} deleted from cluster on server {1}, however following error(s) occurred:\n{2}'
if status:
result = Utils.errorCode(code, reval, params)
raise web.HTTPError(status = "500 Internal Server Error", data = result)
return ''
def resetOptions(clusterName, volumeName):
serverName = getServerName(clusterName)
cmd = 'gluster volume reset ' + volumeName
(status,output) = Utils.executeOnServer(serverName, cmd)
list = output.split(' ')
if (list[-1] == 'exist\n') or (list[-1] == 'exist'):
result = Utils.errorCode('20053', 'volume ' + volumeName + ' does not exist', [])
raise web.HTTPError(status='400 Bad Request', data=result)
isExecSucess(status, output, serverName, volumeName, cmd, '23018',[])
return ''
def setOptions(clusterName, volumeName):
serverName = getServerName(clusterName)
data = web.input()
key = None
value = None
try:
key = data.key.strip()
value = data.value.strip()
except Exception,e:
result = Utils.errorCode('20001', 'key and value are required')
raise web.HTTPError(status = '400 Bad Request', data = result)
def getOptions(clusterName, volumeName):
serverName = getServerName(clusterName)
cmd = 'python ' + BACKEND_SCRIPT + 'get_volume_options.py ' + volumeName
################ execute cmd on local host or remote host
(status,output) = Utils.executeOnServer(serverName, cmd)
list = output.split(' ')
if (list[-1] == 'exist\n') or (list[-1] == 'exist'):
result = Utils.errorCode('20053', 'volume ' + volumeName + ' does not exist', [])
raise web.HTTPError(status='400 Bad Request', data=result)
isExecSucess(status, output, serverName, volumeName, cmd, '23016', [])
return output
def isExecSucess(status, output, serverName, volumeName, cmd, cmdcode, params):
list = output.split(' ')
if (list[-1] == 'exist\n') or (list[-1] == 'exist'):
msg = 'volume {0} does not exist'
params=[]
params.append(volumeName)
result = Utils.errorCode('20053', msg, params)
raise web.HTTPError(status='400 Bad Request', data=result)
if status == -1:
code, reval = '26104', 'error when connecting to remote host {0} from localhost:{1}'
elif status == -2:
code,reval = '26059', 'Error when using pub key to connect remote server {0}.{1}'
elif status == 1:
code, reval = cmdcode, 'Error when executing {0}\n{1}'
if status:
params=[]
params.append(serverName)
params.append(output)
result = Utils.errorCode(code, reval, params)
raise web.HTTPError(status = "500 Internal Server Error", data = result)
def postDeleteTask(clusterName, volumeName):
params=[]
params.append(volumeName)
try:
rebalanceTasks = Globals.db.select('task_info', what = '*', where = 'operation_id=3')
for rebalanceTask in rebalanceTasks:
if rebalanceTask.reference.strip() == volumeName.strip():
Globals.db.delete('task_info', where = 'id=$rebalanceTask.id', vars=locals())
except Exception,e:
code, reval = '23222', 'volume {0} deleted, but error occured when removing rebalance tasks'
result = Utils.errorCode(code, reval, params)
raise web.HTTPError(status = "500 Internal Server Error", data = result)
def rebalanceStop(serverName, volumeName, force=False):
cmd = 'gluster volume rebalance ' + volumeName + ' stop'
if force:
cmd += ' force'
status,output = Utils.executeOnServer(serverName, cmd)
params = []
list = output.split(' ')
if (list[-1] == 'exist') or (list[-1] == 'exist\n'):
params.append(volumeName)
result = Utils.errorCode('20053', 'volume {0} does not exist.\n', params)
raise web.HTTPError(status = '400 Bad Request', data = result)
isExecSucess(status, output, serverName, volumeName, cmd, '23133', params)
web.HTTPError(status='200 OK', data='')
def getServerName(clusterName):
params=[]
if clusterName is None:
result = Utils.errorCode('20002', 'clusterName is required', params)
logger.error("Get serverName: clusterName is null")
raise web.HTTPError(status = '400 Bad Request', data = result)
status, cluster_id, servers = serverService.get_servers(clusterName)
code = None
if status == '-1':
code, reval = '20052', 'No cluster {0}.'
if status == '-2':
code, reval = '20054', 'No server in cluster {0}.'
if code is not None:
params.append(clusterName)
result = Utils.errorCode(code, reval, params)
logger.error(reval)
raise web.HTTPError(status = "400 Bad Request", data = result)
for server in servers:
if Utils.isOnline(server.name):
return server.name.strip()
logger.error("There is no Online Server in cluster[" + clusterName + "]")
params.append(clusterName)
result = Utils.errorCode('22008', 'There is no Online Server in cluster {0}', params)
raise web.HTTPError(status = '400 Bad Request', data = result)
def manageVolume(clusterName,volumeName,data):
serverName = getServerName(clusterName)
if len(serverName) == 0:
return "No server: "+'"'+serverName+'"'
if len(data) == 0:
return 'operation is required'
############### parameters of managin a volume
operation = ''
force = False ### false by defult
############### for rebalance
fixLayout = False ###false by default
migrateData = True ### true by default
forcedDataMigrate = False ### false by default
try:
operation = str(data.operation).strip()
except Exception,e:
result = Utils.errorCode('20001', 'paramters are required\n' + str(e),[])
raise web.HTTPError(status = '400 Bad Request', data = result)
def rebalanceStart(clusterName, serverName, volumeName, fixLayout = False, migrateData = False, forcedDataMigrate = True, force = False):
(status,output) = Utils.rebalanceTaskStart(clusterName, volumeName)
if status != 0:
return output
params = []
cmd = 'gluster volume rebalance ' + volumeName
if (fixLayout == True) and (migrateData == True):
cmd += ' start'
elif fixLayout == True:
cmd += ' fix-layout start'
else:
cmd += ' start'
if force == True:
cmd += 'force'
status,output = Utils.executeOnServer(serverName, cmd)
list = output.split(' ')
if (list[-1] == 'exist') or (list[-1] == 'exist\n'):
params.append(volumeName)
result = Utils.errorCode('20053', 'volume {0} does not exist.\n', params)
raise web.HTTPError(status = '400 Bad Request', data = result)
isExecSucess(status, output, serverName, volumeName, cmd, '23134', params)
web.HTTPError(status='200 OK', data='')
def getLogs(clusterName,volumeName, brickName,linecount='100', severity=None, fromTimeStamp=None, toTimeStamp=None):
params = []
params.append(clusterName)
params.append(volumeName)
params.append(brickName)
params.append(linecount)
params.append(severity)
params.append(fromTimeStamp)
params.append(toTimeStamp)
web.header('Content-Type', 'application/xml')
print severity
if severity not in Globals.VOLUME_LOG_TYPE:
code,reval = '23501','bad severity type:%s' % severity
result = Utils.errorCode(code, reval, params)
web.HTTPError(status = "400 Bad request", data = result)
return result
volLogList = []
if (brickName is not None) and (brickName.strip() is not ''):
server_dir = brickName.split(':/')
if len(server_dir) == 2:
file = server_dir[1].strip()
log = file.replace('/', '-')
LOGFILE = BRICK_LOG_DIR + log + '.log*'
cmd = 'python ' + BACKEND_SCRIPT + 'get_volume_brick_log.py ' + LOGFILE + ' ' + str(linecount)
(status,output) = Utils.executeOnServer(server_dir[0].strip(), cmd)
output.replace('\n',' ')
if status == -1:
code, reval = '26104', 'error when connecting to remote host ' + server_dir[0].strip() + ' from localhost:' + output
elif status == -2:
code,reval = '26059', 'Error when using pub key to connect remote server ' + server_dir[0].strip() + '.' + output
elif status == 1:
if re.match('volume log not found',output):
code, reval = '20053', 'volume ' + volumeName + ' does not exist.\n' + output
result = Utils.errorCode(code, reval, params)
web.HTTPError(status = "400 Bad request", data = result)
return result
else:
code, reval = '23015', 'Error when executing "' + cmd + '".\n' + output
if status:
result = Utils.errorCode(code, reval, params)
web.HTTPError(status = "400 Bad Request", data = '')
return result
weblog = xml.etree.ElementTree.fromstring(output)
for entry in weblog.findall("logMessage"):
volumeLogMessage = VolumeLogMessage()
volumeLogMessage.setBrick(brickName)
volumeLogMessage.setMessage(entry.find('message').text)
volumeLogMessage.setSeverity(entry.find('severity').text)
volumeLogMessage.setTimestamp(entry.find('timestamp').text)
volLogList.append(volumeLogMessage)
else:
cmd = 'python ' + BACKEND_SCRIPT + 'get_volume_bricks.py ' + volumeName
serverName = volumeService.getServerName(clusterName)
(status,output) = Utils.executeOnServer(serverName, cmd)
list = output.split(' ')
if list[-1] == 'exist':
result = Utils.errorCode('20053', 'volume ' + volumeName + ' does not exist', params)
raise web.HTTPError(status='400 Bad Request', data=result)
bricklist = output.split('\n')
VolumeUtils.isExecSucess(status, output, serverName, volumeName, cmd, '23130', params)
#if status == -1:
# code, reval = '26104', 'error when connecting to remote host [' + serverName + '] from localhost:' + output
#elif status == -2:
# code,reval = '26059', 'Error when using pub key to connect remote server [' + serverName + '].' + output
#elif status == 1:
# code, reval = '23019','Error when getting bricks from server [' + serverName + '].'
#result = ''
#if status:
# web.header('Content-Type', 'text/xml')
# result = Utils.errorCode(code, reval, [])
# web.HTTPError(status = "400 Bad Request", data = '')
# return result
for brick in bricklist:
server_dir = brick.split(':/')
if len(server_dir) != 2:
break
file = server_dir[1].strip()
log = file.replace('/', '-')
LOGFILE = BRICK_LOG_DIR + log + '.log'
cmd = 'python ' + BACKEND_SCRIPT + 'get_volume_brick_log.py ' + LOGFILE + ' ' + str(linecount)
status,output = Utils.executeOnServer(server_dir[0].strip(), cmd)
if status == -1:
code, reval = '26104', 'error when connecting to remote host ' + server_dir[0].strip() + ' from localhost:' + output
elif status == -2:
code,reval = '26059', 'Error when using pub key to connect remote server ' + server_dir[0].strip() + '.' + output
elif status == 1:
if re.match('exist', output):
code, reval = '20053', 'volume ' + volumeName + ' does not exist.\n' + output
else:
code, reval = '23021', 'Error when executing "' + cmd + '".\n' + output
if status:
result = Utils.errorCode(code, reval, [])
raise web.HTTPError(status = "400 Bad Request", data = result)
weblog = xml.etree.ElementTree.fromstring(output)
for entry in weblog.findall("logMessage"):
volumeLogMessage = VolumeLogMessage()
volumeLogMessage.setBrick(brick)
volumeLogMessage.setMessage(entry.find('message').text)
volumeLogMessage.setSeverity(entry.find('severity').text)
volumeLogMessage.setTimestamp(entry.find('timestamp').text)
volLogList.append(volumeLogMessage)
if (severity is not None) and (severity.strip() != '') and (severity.lower() != 'all'):
volLogList = filterServerity(volLogList, severity)
if (fromTimeStamp is not None) and (toTimeStamp is not None) and (fromTimeStamp is not '') and (toTimeStamp is not ''):
volLogList = filterTimeStamp(volLogList, fromTimeStamp, toTimeStamp)
responseDom = XmlHandler.ResponseXml()
logMessagesTag = responseDom.appendTagRoute("logMessages")
for volumeLogMessage in volLogList:
logMessageTag = responseDom.createTag("logMessage",None)
logMessagesTag.appendChild(logMessageTag)
logMessageTag.appendChild(responseDom.createTag("brick",volumeLogMessage.getBrick()))
logMessageTag.appendChild(responseDom.createTag("message",volumeLogMessage.getMessage()))
logMessageTag.appendChild(responseDom.createTag("severity",volumeLogMessage.getSeverity()))
logMessageTag.appendChild(responseDom.createTag("timestamp",volumeLogMessage.getTimeStamp()))
return logMessagesTag.toxml()
operation = str(data.operation).strip()
except Exception,e:
result = Utils.errorCode('20001', 'paramters are required\n' + str(e),[])
raise web.HTTPError(status = '400 Bad Request', data = result)
try:
if data.force.lower() == 'true' :
force = True
except Exception,e:
pass
if (operation != 'start') and (operation != 'stop') and (operation != 'cifsConfig')\
and (operation != 'rebalanceStart') and (operation != 'rebalanceStop') and (operation != 'logRotate'):
params = []
params.append(operation)
result = Utils.errorCode('23007', 'invalid operations', params)
raise web.HTTPError(status = '400 Bad Request', data = result)
if operation == 'cifsConfig':
try:
isShared = data.isShared
owner = ''
if isShared == "false":
owner = data.owner
hostAllowed = '\*'
if data.hostAllowed is not None:
hostAllowed = data.hostAllowed
except Exception,e:
pass
if not updateVolumeCIFSUsers(volumeName, isShared, owner, hostAllowed):
params = []
result = Utils.errorCode('20001', 'paramters are required {0}', params)
raise web.HTTPError(status = '400 Bad Request', data = result)
# ################################## get servers in the cluster to create a volume
##################################set args for gluster command
if volumeType.lower() not in VOLUME_TYPE_STR:
code,reval = '23001', 'invalid volume type'
if count is not None:
try:
count_int = int (count)
if (count_int < 0) or (count_int == 0):
code,reval = '23002','invalid count'
except:
code,reval = '23002','invalid count'
if code is not None:
result = Utils.errorCode(code,reval,[])
raise web.HTTPError(status = '400 Bad Request', data = result)
if volumeType.lower() == "replicate" or volumeType.lower() == "distributed replicate":
volumeType = "replica"
elif volumeType.lower() == "stripe" or volumeType == "distributed stripe":
volumeType = "stripe"
# elif volumeType.lower() == 'distributed replicate':
# elif volumeType.lower() == 'distributed stripe':
else:
volumeType = ''
if transportType.upper() == "ETHERNET":
transportType = "tcp"
else:
def removeVolume(clusterName,volumeName):
################################## get servers in the cluster to create a volume
serverName = getServerName(clusterName)
volumeInfo = getVolume(clusterName, volumeName)
# if enableCifs is true, then stop and delete it's cifs configure first
root = ET.fromstring(volumeInfo)
enableCifs = "false"
for subnode in root.getchildren():
if subnode.tag == "enableCifs":
enableCifs = subnode.text
if subnode.tag == "status":
status = subnode.text
if enableCifs == "true":
if status == "ONLINE":
params = []
params.append(volumeName)
result = Utils.errorCode('23101', 'Volume ' + volumeName + ' has been started.Volume needs to be stopped before deletion.', params)
logger.error('Volume ' + volumeName + ' has been started.Volume needs to be stopped before deletion.')
raise web.HTTPError(status = '400 Bad Request', data = result)
if not deleteVolumeCIFSUser(volumeName):
params = []
params.append(volumeName)
result = Utils.errorCode('23204', 'failed to delete cifs configuration while deleting volume: ' + volumeName,params)
logger.error("failed to delete cifs configuration while deleting volume: " + volumeName)
raise web.HTTPError(status = '400 Bad Request', data = result)
cmd = 'python ' + BACKEND_SCRIPT + 'get_volume_bricks.py ' + volumeName
(status,output) = Utils.executeOnServer(serverName, cmd)
list = output.split(' ')
if (list[-1] == 'exist\n') or (list[-1] == 'exist'):
params = []
params.append(volumeName)
result = Utils.errorCode('20053', 'volume {0} does not exist', params)
logger.error("Remove volume: volume[" + volumeName + "] does not exist")
raise web.HTTPError(status='400 Bad Request', data=result)
isExecSucess(status, output, serverName, volumeName, cmd, '23019',[])
################################## prepare the command
brickList = output.split('\n')
cmd = 'gluster --mode=script volume delete ' + volumeName
################################## execute on remote host or local host
status,output = Utils.executeOnServer(serverName, cmd)
params = []
params.append(clusterName)
params.append(volumeName)
isExecSucess(status, output, serverName, volumeName, cmd, '23101', params)
deleteFlag = False
postDelete(volumeName, brickList,deleteFlag)
try:
postDeleteTask(clusterName, volumeName)
except:
code, reval = '23101', 'Volume ' + volumeName +' deleted from cluster, however following error(s) occurred:\n' + output
if status:
result = Utils.errorCode(code, reval, params)
logger.error(reval)
raise web.HTTPError(status = "400 Bad Request", data = result)
try:
Globals.db.delete('volume_info',where='name=$volumeName', vars=locals())
except Exception,e:
result = Utils.errorCode('23101','error while deleting volume from DB',parseString)
logger.error(reval)
raise web.HTTPError(status = "400 Bad Request", data = result)
