def createVolume(clusterName,data): serverName = getServerName(clusterName) cmd, options, volumeName = preCreateVolume(data) volume_info = Globals.db.select('volume_info', what='*', where='name=$volumeName', vars=locals()) if len(volume_info) > 0: params = [] params.append(volumeName) result = Utils.errorCode('23015','error while adding a volume:volume ' + volumeName + ' already exists', params) logger.error('error while adding a volume:volume ' + volumeName + ' already exists') raise web.HTTPError(status = "400 Bad Request", data = result) #################################execute on localhost or remote host status,output = Utils.executeOnServer(serverName, cmd) if status == -1: code, reval = '26104', 'error when connecting to remote host ' + serverName + ' from localhost:' + output elif status == -2: code,reval = '26059', 'Error when using pub key to connect remote server ' + serverName + '.' + output elif status == 1: if re.match('exists', output): code, reval = '20103', output else: code, reval = '23015', 'Error when creating volume' + volumeName + '.' + output if status: result = Utils.errorCode(code, reval, []) logger.error(reval) raise web.HTTPError(status = "400 Bad Request", data = result) optionToken = options.split(",") hostAllowed = "\*" for option in optionToken: if option.split("=")[0] == "auth.allow": hostAllowed = option.split("=")[1] nasProtocols = data.accessProtocols if "CIFS" in nasProtocols: isShared = data.isShared owner = data.owner if not createVolumeCIFSUsers(volumeName, isShared, owner, hostAllowed): params = [] params.append(volumeName) result = Utils.errorCode('23201', 'failed to configure cifs for volume: %s' % volumeName, params) Utils.executeOnServer(serverName, 'gluster --mode=script volume delete ' + volumeName) logger.error("failed to configure cifs for volume:" + volumeName) raise web.HTTPError(status = '400 Bad Request',data = result) try: cluster = Globals.db.select('cluster_info',where='name=$clusterName',vars=locals()) clusterid=cluster[0].id Globals.db.insert('volume_info',name=volumeName,cluster_id=clusterid) except: params = [] params.append(volumeName) result = Utils.errorCode('23015','error while adding a volume:error while inserting volume_info into DB', params) deleteVolumeCIFSUser(volumeName) Utils.executeOnServer(serverName, 'gluster --mode=script volume delete ' + volumeName) #postDelete(volumeName, brickList,deleteFlag) logger.error('error while adding a volume:error while inserting volume_info into DB') raise web.HTTPError(status = '400 Bad Request',data = result) return None
def downloadLogs(clusterName, volumeName): params = [] web.header('Content-Type','application/octet-stream') serverName = getServerName(clusterName) hostName = os.popen('hostname').read() cmd = 'python ' + BACKEND_SCRIPT + 'get_volume_bricks.py ' + volumeName (status,output) = Utils.executeOnServer(serverName, cmd) list = output.split(' ') if (list[-1] == 'exist\n') or (list[-1] == 'exist'): params.append(volumeName) result = Utils.errorCode('20053', 'volume ' + volumeName + ' does not exist', params) raise web.HTTPError(status='400 Bad Request', data=result) isExecSucess(status, output, serverName, volumeName, cmd, '23019',[]) bricklist = output.split('\n') if status != 0: return output result = '' if os.path.exists(TMP_LOG_DIR) == False: os.mkdir(TMP_LOG_DIR) logs = '' for brick in bricklist: server_dir = brick.split(':/') if len(server_dir) != 2: break file = server_dir[1].strip() log = file.replace('/', '-') + '.log' status,files = Utils.executeOnServer(server_dir[0].strip(),'ls ' + BRICK_LOG_DIR +'|grep ^' + log) files = files.split('\n') for file in files: if file.strip() is '': break cmd = 'scp ' + str(BRICK_LOG_DIR) + str(file) + \ ' root@' + str(hostName).strip().split('\n')[0] + \ ':' + TMP_LOG_DIR + '/' + str(server_dir[0].strip()) + '-' + file (status,output) = Utils.executeOnServer(server_dir[0].strip(), cmd) logs += ' ' + server_dir[0].strip() + '-' + file if output[:7] == 'Warning': continue isExecSucess(status, output, serverName, volumeName, cmd, '23019', []) if os.path.exists(TMP_LOG_DIR + volumeName + '.tar.gz'): os.remove(TMP_LOG_DIR + volumeName + '.tar.gz') os.chdir(TMP_LOG_DIR) log = TMP_LOG_DIR + volumeName + '.tar.gz' if os.path.exists(log): os.remove(log) os.popen('tar cvzf ' + volumeName + '.tar.gz ' + logs) os.popen('rm -rf ' + logs) try: f=open(volumeName+'.tar.gz') stream = f.read() except Exception,e: params.append(volumeName) result = Utils.errorCode('23019', 'error when downloading logs', params) raise web.HTTPError(status='500 Internal Error', data=result)
def preCreateVolume(data): if len(data) == 0: return 'clusterName is required' code = None count = None try: volumeName = data.volumeName volumeType = data.volumeType transportType = data.transportType if data.replicaCount != '0': count = data.replicaCount if data.stripeCount != '0': count = data.stripeCount # # # accessProtocols = data.accessProtocols options = data.options bricks = data.bricks nasProtocols = data.accessProtocols if 'CIFS' in nasProtocols: isShared = data.isShared owner = data.owner except Exception,e: params = [] params.append(str(e)) result = Utils.errorCode('20001', 'paramters are required {0}', params) raise web.HTTPError(status = '400 Bad Request', data = result)
def postDelete(volumeName, brickList,deleteFlag = False): params = [] params.append(volumeName) params.append(brickList) params.append(deleteFlag) for brick in brickList: if brick.strip() is '': continue cmd = 'python ' + BACKEND_SCRIPT + 'clear_volume_directory.py' server_dir = brick.split(":/") if len(server_dir) != 2: break cmd += ' /' + server_dir[1].strip() status,output = Utils.executeOnServer(server_dir[0].strip(), cmd) if status == -1: params = [] params.append(volumeName) params.append(server_dir[0].strip()) params.append(output) code, reval = '26104', 'Volume {0} deleted from cluster, however following error(s) occurred:\nerror when connecting to remote host {1} from localhost:{2}' elif status == -2: code,reval = '26059', 'Volume {0} deleted from cluster, however following error(s) occurred:\nError when using pub key to connect remote server {1}.{2}' elif status == 1: if re.match('exist', output) or re.match('exists', output) or re.match('exist\n',output): code, reval = '20053', 'volume {0} does not exist.\n' else: code, reval = '23101', 'Volume {0} deleted from cluster on server {1}, however following error(s) occurred:\n{2}' if status: result = Utils.errorCode(code, reval, params) raise web.HTTPError(status = "500 Internal Server Error", data = result) return ''
def resetOptions(clusterName, volumeName): serverName = getServerName(clusterName) cmd = 'gluster volume reset ' + volumeName (status,output) = Utils.executeOnServer(serverName, cmd) list = output.split(' ') if (list[-1] == 'exist\n') or (list[-1] == 'exist'): result = Utils.errorCode('20053', 'volume ' + volumeName + ' does not exist', []) raise web.HTTPError(status='400 Bad Request', data=result) isExecSucess(status, output, serverName, volumeName, cmd, '23018',[]) return ''
def setOptions(clusterName, volumeName): serverName = getServerName(clusterName) data = web.input() key = None value = None try: key = data.key.strip() value = data.value.strip() except Exception,e: result = Utils.errorCode('20001', 'key and value are required') raise web.HTTPError(status = '400 Bad Request', data = result)
def getOptions(clusterName, volumeName): serverName = getServerName(clusterName) cmd = 'python ' + BACKEND_SCRIPT + 'get_volume_options.py ' + volumeName ################ execute cmd on local host or remote host (status,output) = Utils.executeOnServer(serverName, cmd) list = output.split(' ') if (list[-1] == 'exist\n') or (list[-1] == 'exist'): result = Utils.errorCode('20053', 'volume ' + volumeName + ' does not exist', []) raise web.HTTPError(status='400 Bad Request', data=result) isExecSucess(status, output, serverName, volumeName, cmd, '23016', []) return output
def isExecSucess(status, output, serverName, volumeName, cmd, cmdcode, params): list = output.split(' ') if (list[-1] == 'exist\n') or (list[-1] == 'exist'): msg = 'volume {0} does not exist' params=[] params.append(volumeName) result = Utils.errorCode('20053', msg, params) raise web.HTTPError(status='400 Bad Request', data=result) if status == -1: code, reval = '26104', 'error when connecting to remote host {0} from localhost:{1}' elif status == -2: code,reval = '26059', 'Error when using pub key to connect remote server {0}.{1}' elif status == 1: code, reval = cmdcode, 'Error when executing {0}\n{1}' if status: params=[] params.append(serverName) params.append(output) result = Utils.errorCode(code, reval, params) raise web.HTTPError(status = "500 Internal Server Error", data = result)
def postDeleteTask(clusterName, volumeName): params=[] params.append(volumeName) try: rebalanceTasks = Globals.db.select('task_info', what = '*', where = 'operation_id=3') for rebalanceTask in rebalanceTasks: if rebalanceTask.reference.strip() == volumeName.strip(): Globals.db.delete('task_info', where = 'id=$rebalanceTask.id', vars=locals()) except Exception,e: code, reval = '23222', 'volume {0} deleted, but error occured when removing rebalance tasks' result = Utils.errorCode(code, reval, params) raise web.HTTPError(status = "500 Internal Server Error", data = result)
def rebalanceStop(serverName, volumeName, force=False): cmd = 'gluster volume rebalance ' + volumeName + ' stop' if force: cmd += ' force' status,output = Utils.executeOnServer(serverName, cmd) params = [] list = output.split(' ') if (list[-1] == 'exist') or (list[-1] == 'exist\n'): params.append(volumeName) result = Utils.errorCode('20053', 'volume {0} does not exist.\n', params) raise web.HTTPError(status = '400 Bad Request', data = result) isExecSucess(status, output, serverName, volumeName, cmd, '23133', params) web.HTTPError(status='200 OK', data='')
def getServerName(clusterName): params=[] if clusterName is None: result = Utils.errorCode('20002', 'clusterName is required', params) logger.error("Get serverName: clusterName is null") raise web.HTTPError(status = '400 Bad Request', data = result) status, cluster_id, servers = serverService.get_servers(clusterName) code = None if status == '-1': code, reval = '20052', 'No cluster {0}.' if status == '-2': code, reval = '20054', 'No server in cluster {0}.' if code is not None: params.append(clusterName) result = Utils.errorCode(code, reval, params) logger.error(reval) raise web.HTTPError(status = "400 Bad Request", data = result) for server in servers: if Utils.isOnline(server.name): return server.name.strip() logger.error("There is no Online Server in cluster[" + clusterName + "]") params.append(clusterName) result = Utils.errorCode('22008', 'There is no Online Server in cluster {0}', params) raise web.HTTPError(status = '400 Bad Request', data = result)
def manageVolume(clusterName,volumeName,data): serverName = getServerName(clusterName) if len(serverName) == 0: return "No server: "+'"'+serverName+'"' if len(data) == 0: return 'operation is required' ############### parameters of managin a volume operation = '' force = False ### false by defult ############### for rebalance fixLayout = False ###false by default migrateData = True ### true by default forcedDataMigrate = False ### false by default try: operation = str(data.operation).strip() except Exception,e: result = Utils.errorCode('20001', 'paramters are required\n' + str(e),[]) raise web.HTTPError(status = '400 Bad Request', data = result)
def rebalanceStart(clusterName, serverName, volumeName, fixLayout = False, migrateData = False, forcedDataMigrate = True, force = False): (status,output) = Utils.rebalanceTaskStart(clusterName, volumeName) if status != 0: return output params = [] cmd = 'gluster volume rebalance ' + volumeName if (fixLayout == True) and (migrateData == True): cmd += ' start' elif fixLayout == True: cmd += ' fix-layout start' else: cmd += ' start' if force == True: cmd += 'force' status,output = Utils.executeOnServer(serverName, cmd) list = output.split(' ') if (list[-1] == 'exist') or (list[-1] == 'exist\n'): params.append(volumeName) result = Utils.errorCode('20053', 'volume {0} does not exist.\n', params) raise web.HTTPError(status = '400 Bad Request', data = result) isExecSucess(status, output, serverName, volumeName, cmd, '23134', params) web.HTTPError(status='200 OK', data='')
def getLogs(clusterName,volumeName, brickName,linecount='100', severity=None, fromTimeStamp=None, toTimeStamp=None): params = [] params.append(clusterName) params.append(volumeName) params.append(brickName) params.append(linecount) params.append(severity) params.append(fromTimeStamp) params.append(toTimeStamp) web.header('Content-Type', 'application/xml') print severity if severity not in Globals.VOLUME_LOG_TYPE: code,reval = '23501','bad severity type:%s' % severity result = Utils.errorCode(code, reval, params) web.HTTPError(status = "400 Bad request", data = result) return result volLogList = [] if (brickName is not None) and (brickName.strip() is not ''): server_dir = brickName.split(':/') if len(server_dir) == 2: file = server_dir[1].strip() log = file.replace('/', '-') LOGFILE = BRICK_LOG_DIR + log + '.log*' cmd = 'python ' + BACKEND_SCRIPT + 'get_volume_brick_log.py ' + LOGFILE + ' ' + str(linecount) (status,output) = Utils.executeOnServer(server_dir[0].strip(), cmd) output.replace('\n',' ') if status == -1: code, reval = '26104', 'error when connecting to remote host ' + server_dir[0].strip() + ' from localhost:' + output elif status == -2: code,reval = '26059', 'Error when using pub key to connect remote server ' + server_dir[0].strip() + '.' + output elif status == 1: if re.match('volume log not found',output): code, reval = '20053', 'volume ' + volumeName + ' does not exist.\n' + output result = Utils.errorCode(code, reval, params) web.HTTPError(status = "400 Bad request", data = result) return result else: code, reval = '23015', 'Error when executing "' + cmd + '".\n' + output if status: result = Utils.errorCode(code, reval, params) web.HTTPError(status = "400 Bad Request", data = '') return result weblog = xml.etree.ElementTree.fromstring(output) for entry in weblog.findall("logMessage"): volumeLogMessage = VolumeLogMessage() volumeLogMessage.setBrick(brickName) volumeLogMessage.setMessage(entry.find('message').text) volumeLogMessage.setSeverity(entry.find('severity').text) volumeLogMessage.setTimestamp(entry.find('timestamp').text) volLogList.append(volumeLogMessage) else: cmd = 'python ' + BACKEND_SCRIPT + 'get_volume_bricks.py ' + volumeName serverName = volumeService.getServerName(clusterName) (status,output) = Utils.executeOnServer(serverName, cmd) list = output.split(' ') if list[-1] == 'exist': result = Utils.errorCode('20053', 'volume ' + volumeName + ' does not exist', params) raise web.HTTPError(status='400 Bad Request', data=result) bricklist = output.split('\n') VolumeUtils.isExecSucess(status, output, serverName, volumeName, cmd, '23130', params) #if status == -1: # code, reval = '26104', 'error when connecting to remote host [' + serverName + '] from localhost:' + output #elif status == -2: # code,reval = '26059', 'Error when using pub key to connect remote server [' + serverName + '].' + output #elif status == 1: # code, reval = '23019','Error when getting bricks from server [' + serverName + '].' #result = '' #if status: # web.header('Content-Type', 'text/xml') # result = Utils.errorCode(code, reval, []) # web.HTTPError(status = "400 Bad Request", data = '') # return result for brick in bricklist: server_dir = brick.split(':/') if len(server_dir) != 2: break file = server_dir[1].strip() log = file.replace('/', '-') LOGFILE = BRICK_LOG_DIR + log + '.log' cmd = 'python ' + BACKEND_SCRIPT + 'get_volume_brick_log.py ' + LOGFILE + ' ' + str(linecount) status,output = Utils.executeOnServer(server_dir[0].strip(), cmd) if status == -1: code, reval = '26104', 'error when connecting to remote host ' + server_dir[0].strip() + ' from localhost:' + output elif status == -2: code,reval = '26059', 'Error when using pub key to connect remote server ' + server_dir[0].strip() + '.' + output elif status == 1: if re.match('exist', output): code, reval = '20053', 'volume ' + volumeName + ' does not exist.\n' + output else: code, reval = '23021', 'Error when executing "' + cmd + '".\n' + output if status: result = Utils.errorCode(code, reval, []) raise web.HTTPError(status = "400 Bad Request", data = result) weblog = xml.etree.ElementTree.fromstring(output) for entry in weblog.findall("logMessage"): volumeLogMessage = VolumeLogMessage() volumeLogMessage.setBrick(brick) volumeLogMessage.setMessage(entry.find('message').text) volumeLogMessage.setSeverity(entry.find('severity').text) volumeLogMessage.setTimestamp(entry.find('timestamp').text) volLogList.append(volumeLogMessage) if (severity is not None) and (severity.strip() != '') and (severity.lower() != 'all'): volLogList = filterServerity(volLogList, severity) if (fromTimeStamp is not None) and (toTimeStamp is not None) and (fromTimeStamp is not '') and (toTimeStamp is not ''): volLogList = filterTimeStamp(volLogList, fromTimeStamp, toTimeStamp) responseDom = XmlHandler.ResponseXml() logMessagesTag = responseDom.appendTagRoute("logMessages") for volumeLogMessage in volLogList: logMessageTag = responseDom.createTag("logMessage",None) logMessagesTag.appendChild(logMessageTag) logMessageTag.appendChild(responseDom.createTag("brick",volumeLogMessage.getBrick())) logMessageTag.appendChild(responseDom.createTag("message",volumeLogMessage.getMessage())) logMessageTag.appendChild(responseDom.createTag("severity",volumeLogMessage.getSeverity())) logMessageTag.appendChild(responseDom.createTag("timestamp",volumeLogMessage.getTimeStamp())) return logMessagesTag.toxml()
operation = str(data.operation).strip() except Exception,e: result = Utils.errorCode('20001', 'paramters are required\n' + str(e),[]) raise web.HTTPError(status = '400 Bad Request', data = result) try: if data.force.lower() == 'true' : force = True except Exception,e: pass if (operation != 'start') and (operation != 'stop') and (operation != 'cifsConfig')\ and (operation != 'rebalanceStart') and (operation != 'rebalanceStop') and (operation != 'logRotate'): params = [] params.append(operation) result = Utils.errorCode('23007', 'invalid operations', params) raise web.HTTPError(status = '400 Bad Request', data = result) if operation == 'cifsConfig': try: isShared = data.isShared owner = '' if isShared == "false": owner = data.owner hostAllowed = '\*' if data.hostAllowed is not None: hostAllowed = data.hostAllowed except Exception,e: pass if not updateVolumeCIFSUsers(volumeName, isShared, owner, hostAllowed): params = []
result = Utils.errorCode('20001', 'paramters are required {0}', params) raise web.HTTPError(status = '400 Bad Request', data = result) # ################################## get servers in the cluster to create a volume ##################################set args for gluster command if volumeType.lower() not in VOLUME_TYPE_STR: code,reval = '23001', 'invalid volume type' if count is not None: try: count_int = int (count) if (count_int < 0) or (count_int == 0): code,reval = '23002','invalid count' except: code,reval = '23002','invalid count' if code is not None: result = Utils.errorCode(code,reval,[]) raise web.HTTPError(status = '400 Bad Request', data = result) if volumeType.lower() == "replicate" or volumeType.lower() == "distributed replicate": volumeType = "replica" elif volumeType.lower() == "stripe" or volumeType == "distributed stripe": volumeType = "stripe" # elif volumeType.lower() == 'distributed replicate': # elif volumeType.lower() == 'distributed stripe': else: volumeType = '' if transportType.upper() == "ETHERNET": transportType = "tcp" else:
def removeVolume(clusterName,volumeName): ################################## get servers in the cluster to create a volume serverName = getServerName(clusterName) volumeInfo = getVolume(clusterName, volumeName) # if enableCifs is true, then stop and delete it's cifs configure first root = ET.fromstring(volumeInfo) enableCifs = "false" for subnode in root.getchildren(): if subnode.tag == "enableCifs": enableCifs = subnode.text if subnode.tag == "status": status = subnode.text if enableCifs == "true": if status == "ONLINE": params = [] params.append(volumeName) result = Utils.errorCode('23101', 'Volume ' + volumeName + ' has been started.Volume needs to be stopped before deletion.', params) logger.error('Volume ' + volumeName + ' has been started.Volume needs to be stopped before deletion.') raise web.HTTPError(status = '400 Bad Request', data = result) if not deleteVolumeCIFSUser(volumeName): params = [] params.append(volumeName) result = Utils.errorCode('23204', 'failed to delete cifs configuration while deleting volume: ' + volumeName,params) logger.error("failed to delete cifs configuration while deleting volume: " + volumeName) raise web.HTTPError(status = '400 Bad Request', data = result) cmd = 'python ' + BACKEND_SCRIPT + 'get_volume_bricks.py ' + volumeName (status,output) = Utils.executeOnServer(serverName, cmd) list = output.split(' ') if (list[-1] == 'exist\n') or (list[-1] == 'exist'): params = [] params.append(volumeName) result = Utils.errorCode('20053', 'volume {0} does not exist', params) logger.error("Remove volume: volume[" + volumeName + "] does not exist") raise web.HTTPError(status='400 Bad Request', data=result) isExecSucess(status, output, serverName, volumeName, cmd, '23019',[]) ################################## prepare the command brickList = output.split('\n') cmd = 'gluster --mode=script volume delete ' + volumeName ################################## execute on remote host or local host status,output = Utils.executeOnServer(serverName, cmd) params = [] params.append(clusterName) params.append(volumeName) isExecSucess(status, output, serverName, volumeName, cmd, '23101', params) deleteFlag = False postDelete(volumeName, brickList,deleteFlag) try: postDeleteTask(clusterName, volumeName) except: code, reval = '23101', 'Volume ' + volumeName +' deleted from cluster, however following error(s) occurred:\n' + output if status: result = Utils.errorCode(code, reval, params) logger.error(reval) raise web.HTTPError(status = "400 Bad Request", data = result) try: Globals.db.delete('volume_info',where='name=$volumeName', vars=locals()) except Exception,e: result = Utils.errorCode('23101','error while deleting volume from DB',parseString) logger.error(reval) raise web.HTTPError(status = "400 Bad Request", data = result)