def upload_folder(sftp: pysftp.Connection, from_computer: str, to_server: str, is_last: bool = False, step: int = 1, blanks: List[bool] = None) -> None: """ Copy a file or a folder :param sftp: The sftp connection :param from_computer: The file or folder to copy :param to_server: The folder to copy the file/folder inside :param is_last: The boolean that say if it is the last file/folder :param step: The number of space to space printing :param blanks: List of booleans that say if the repository is empty """ if blanks is None: blanks = [False] if len(blanks) <= step // 4: blanks.append(False) # Print space and tree before folder for i in range(step // 4): if blanks[i]: print(' ' * 4, end="") else: print('| ', end="") if is_last: print('└───', end="") blanks[step // 4] = True else: print('├───', end="") # Change current name to_server += '/' + from_computer.split('/')[-1] # Check the type of the path if os.path.isfile(from_computer): # We are in case of a file, so we can upload it sftp.put(from_computer, to_server) print(f" File '{from_computer.split('/')[-1]}' uploaded") else: # We are in case of a folder sftp.execute(f"mkdir {to_server}") print(f" Folder '{from_computer.split('/')[-1]}' created") index = 1 for item in os.listdir(from_computer): upload_folder(sftp, f"{from_computer}/{item}", to_server, len(os.listdir(from_computer)) == index, step + 4, blanks) index += 1 blanks[step // 4] = False
def uploadFilesToPath(fls: list, targetPath: str, sftpConnection: sftp.Connection): print('Uploading files: ' + str(fls)) for f in fls: if (f in ignore): print('File ignored: ' + f) continue destF = f.split('\\')[-1] try: sftpConnection.put(localpath=f, remotepath=targetPath + destF, confirm=True) except Exception as e: print('Unable to upload file: ' + str(f) + '\nError: ' + str(e))
def put(sftp: pysftp.Connection, filename: str): try: sftp.put(filename, os.path.basename(filename), preserve_mtime=False) return True except FileNotFoundError: print("Error:", FILE_NOT_FOUND_ERROR_MESSAGE) return False except OSError: if sftp.isdir(filename): # The file we want to upload is a directory on remote. # pysftp doesn't handle this case well, so we'll print a helpful # error message of our own. print("Error:", FOLDER_CONFLICT_ERROR_MESSAGE) return False else: # It's some other error; we'll let the main handler deal with it. raise
class SFTP: def __init__(self, sftp_server_config): self.server = sftp_server_config['host_address'] self.username = sftp_server_config['user_name'] self.private_key = sftp_server_config['key_path'] self.sftp_folder = sftp_server_config['sftp_folder'] self.connection_opts = CnOpts() self.connection_opts.hostkeys = None self.__connection: Connection = None def connect(self): if not self.__connection: self.__connection = Connection(self.server, self.username, self.private_key, cnopts=self.connection_opts) return self def put(self, file, remote_path): return self.__connection.put(file, remote_path) def remove(self, path): return self.__connection.remove(path) def rmdir(self, path): return self.__connection.rmdir(path) def listdir(self, path): return self.__connection.listdir(path) def is_dir(self, path): return self.__connection.isdir(path) def is_file(self, path): return self.__connection.isfile(path) def close(self): self.__connection.close() self.__connection = None
class PollServer(): """ This is the controller that calls/runs scripts on a Saturn server as required by saturnring """ def __init__(self, serverDNS): """ The init script for the class """ self.serverDNS = str(serverDNS) BASE_DIR = dirname(dirname(__file__)) config = ConfigReader() self.userName = config.get('saturnnode', 'user') self.keyFile = join(BASE_DIR, config.get('saturnring', 'privatekeyfile')) self.rembashpath = config.get('saturnnode', 'bashpath') self.rempypath = config.get('saturnnode', 'pythonpath') self.iscsiconfdir = join(BASE_DIR, config.get('saturnring', 'iscsiconfigdir')) self.remoteinstallLoc = config.get('saturnnode', 'install_location') self.localbashscripts = join(BASE_DIR, config.get('saturnring', 'bashscripts')) try: self.srv = Connection(self.serverDNS, self.userName, self.keyFile) except: logger.critical( "Failed SSH-exec connection on Saturn server %s; possible cause: %s" % (self.serverDNS, format_exc())) def InstallScripts(self): """ Copy bash scripts from the saturnringserver into the saturn server via sftp """ #srv = Connection(self.serverDNS,self.userName,self.keyFile) self.srv.execute('mkdir -p ' + self.remoteinstallLoc + 'saturn-bashscripts/') self.srv.chdir(self.remoteinstallLoc + 'saturn-bashscripts/') locallist = listdir(self.localbashscripts) for localfile in locallist: self.srv.put(self.localbashscripts + localfile) self.srv.execute("chmod 777 " + self.remoteinstallLoc + 'saturn-bashscripts/' + localfile) #srv.close() logger.info("Installed scripts") def Exec(self, command): """ Helper function for executing a remote command over an SSH tunnel """ rtncmd = -1 try: #srv = Connection(self.serverDNS,self.userName,self.keyFile) rtncmd = self.srv.execute(command) #srv.close() except: logger.error("Failed SSH-exec command: %s on Saturn server %s" % (command, self.serverDNS)) logger.error(format_exc()) return rtncmd def ParseLVM(self, strList, delimitStr, paraList): """ Parse lvdisplay and vgdisplay strings and populate dictionaries with relevant information """ rtnDict = {} valueDict = {} for aLine in strList: if (delimitStr in aLine): if len(valueDict) == len(paraList): rtnDict[valueDict[paraList[0]]] = valueDict valueDict = {} continue else: for anItem in paraList: if anItem in aLine: valueDict[anItem] = aLine.split(anItem)[1].strip() if '%' in valueDict[anItem]: valueDict[anItem] = float(valueDict[anItem][:-2]) continue if '/' in valueDict[anItem]: valueDict[anItem] = valueDict[anItem].split('/')[0] if 'GiB' in valueDict[anItem]: valueDict[anItem] = float( valueDict[anItem].split('GiB')[0]) * 1 continue if 'MiB' in valueDict[anItem]: valueDict[anItem] = float( valueDict[anItem].split('MiB')[0]) * 0.001 continue continue if len(valueDict) == len(paraList): rtnDict[valueDict[paraList[0]]] = valueDict logger.info(rtnDict) return rtnDict def UpdateLVs(self, vgObject): """ Update LV information, called to monitor and update capacity. """ lvdict = self.GetLVs(vgObject.vguuid) if "No LVs " in lvdict: logger.info( "There are no LVs in %s to run UpdateLVs on in Saturn host %s" % (vgObject.vguuid, self.serverDNS)) return 0 if lvdict == -1: logger.error( "Could not run GetLVs (perhaps there are no LVs in this VG yet?)" ) return -1 lvs = LV.objects.filter(vg=vgObject) for lvName, lvinfo in lvdict.iteritems(): if len(lvs.filter(lvname=lvName)): preexistLV = lvs.filter(lvname=lvName)[0] preexistLV.lvsize = lvinfo['LV Size'] preexistLV.save(update_fields=['lvsize']) else: logger.warn("Found orphan LV %s in VG %s on host %s" % (lvName, vgObject.vguuid, self.serverDNS)) def GetLVs(self, vguuid): """ Wrapper for parselvm (for LVs), actually populating the DB is done by the UpdateLV function """ execCmd = " ".join([ 'sudo', 'vgdisplay', '-c', '|', 'grep', vguuid, '|', 'cut -d: -f1' ]) vgname = self.Exec(execCmd)[0].strip() if vgname == -1: logger.error("Could not execute %s on %s " % (execCmd, self.serverDNS)) return -1 execCmd = " ".join(['sudo', 'lvdisplay', '--units g', vgname]) lvStrList = self.Exec(execCmd) if lvStrList == [""]: return "No LVs in %s" % (vguuid, ) if lvStrList == -1: logger.error("Could not execute %s on %s " % (execCmd, self.serverDNS)) return -1 delimitStr = '--- Logical volume ---' paraList = ['LV Name', 'LV UUID', 'LV Size'] lvs = self.ParseLVM(lvStrList, delimitStr, paraList) return lvs def GetVG(self): #Unit test this again """ Wrapper for parseLVM (for VGs)+populating the DB """ delimitStr = '--- Volume group ---' paraList = [ 'VG Name', 'VG Size', 'PE Size', 'Total PE', 'Free PE / Size', 'VG UUID' ] execCmd = " ".join(['sudo', 'vgdisplay', '--units g']) vgStrList = self.Exec(execCmd) if vgStrList == -1: return -1 vgs = self.ParseLVM(vgStrList, delimitStr, paraList) #logger.info("VGStating on %s returns %s " % (self.serverDNS, str(vgs)) ) rtnvguuidList = "" for vgname in vgs: try: execCmd = " ".join([ 'sudo', self.remoteinstallLoc + 'saturn-bashscripts/vgstats.sh', vgname ]) cmdStr = self.Exec(execCmd) logger.info(self.serverDNS + ": " + " ".join([ 'sudo', self.rembashpath, self.remoteinstallLoc + 'saturn-bashscripts/vgstats.sh', vgname ]) + ': returned ' + str(cmdStr)) maxavl = float(cmdStr[0].rstrip()) totalGB = float(cmdStr[1].rstrip()) isThin = bool(int(cmdStr[2].rstrip())) except: logger.warn("Unable to run VGscan, disabling VG on " + self.serverDNS) logger.warn(format_exc()) try: vg = VG.objects.get(vguuid=vgs[vgname]['VG UUID']) vg.in_error = True vg.save(update_fields=['in_error']) except: logger.error("VG not found in DB: %s" % (vgs[vgname]['VG UUID'], )) return 3 existingvgs = VG.objects.filter(vguuid=vgs[vgname]['VG UUID']) if len(existingvgs) == 1: existingvg = existingvgs[0] existingvg.in_error = False existingvg.CurrentAllocGB = totalGB - maxavl #Target.objects.filter(targethost=existingvg.vghost).aggregate(Sum('sizeinGB'))['sizeinGB__sum'] existingvg.totalGB = totalGB existingvg.maxavlGB = maxavl existingvg.is_thin = isThin existingvg.vgsize = vgs[vgname]['VG Size'] existingvg.save(update_fields=[ 'totalGB', 'maxavlGB', 'vgsize', 'CurrentAllocGB', 'in_error', 'is_thin' ]) logger.info("Ran in existingVG loop") else: logger.info("Found new VG, adding\n" + str(vgs[vgname])) myvg = VG( vghost=StorageHost.objects.get(dnsname=self.serverDNS), vgsize=vgs[vgname]['VG Size'], vguuid=vgs[vgname]['VG UUID'], vgpesize=vgs[vgname]['PE Size'], vgtotalpe=vgs[vgname]['Total PE'], vgfreepe=vgs[vgname]['Free PE / Size'], totalGB=totalGB, maxavlGB=maxavl, is_thin=isThin) myvg.save() #force_update=True) rtnvguuidList = rtnvguuidList + ',' + vgs[vgname]['VG UUID'] return rtnvguuidList[1:] def GitSave(self, vguuid, commentStr): """ Check in changes to config files into git repository """ try: #srv = Connection(self.serverDNS,self.userName,self.keyFile) self.srv.get('/temp/scst.conf', self.iscsiconfdir + self.serverDNS + '.scst.conf') self.srv.get( '/temp/' + vguuid, self.iscsiconfdir + self.serverDNS + '.' + vguuid + '.lvm') try: repo = Repo(self.iscsiconfdir) filelist = [ f for f in listdir(self.iscsiconfdir) if isfile(join(self.iscsiconfdir, f)) ] repo.stage(filelist) repo.do_commit(commentStr) except: var = format_exc() logger.error("During GitSave: %s: Git save error: %s" % (commentStr, var)) except: var = format_exc() logger.error("During GitSave: %s: PYSFTP download error: %s" % (commentStr, var)) def CreateTarget(self, iqnTarget, iqnInit, sizeinGB, storageip1, storageip2, vguuid): """ Create iSCSI target by running the createtarget script; and save latest scst.conf from the remote server (overwrite) """ #self.srv = Connection(self.serverDNS,self.userName,self.keyFile) cmdStr = " ".join([ 'sudo', self.rembashpath, self.remoteinstallLoc + 'saturn-bashscripts/createtarget.sh', str(sizeinGB), iqnTarget, storageip1, storageip2, iqnInit, vguuid ]) #srv.close() logger.info("Launching createtarget with \n%s" % (cmdStr, )) exStr = self.Exec(cmdStr) if exStr == -1: return -1 commentStr = "Trying to create target %s " % (iqnTarget, ) self.GitSave(vguuid, commentStr) logger.info("Execution report for %s: %s" % (cmdStr, "\t".join(exStr))) if "SUCCESS" in str(exStr): logger.info("Returning successful createtarget run") return 1 else: logger.error("Returning failed createtarget run") return 0 def GetTargetsState(self): """ Read targets to determine their latest state via the parsetarget script """ cmdStr = " ".join([ "sudo", self.rempypath, self.remoteinstallLoc + 'saturn-bashscripts/parsetarget.py' ]) exStr = self.Exec(cmdStr) if exStr == -1: return -1 for eachLine in exStr: iqntar = eachLine.split()[0] tar = Target.objects.filter(iqntar=iqntar) if len(tar) == 1: tar = tar[0] if "no session" in eachLine: tar.sessionup = False tar.rkbpm = 0 tar.wkbpm = 0 else: tar.sessionup = True rkb = long(eachLine.split()[1]) tar.rkbpm = long(rkb - tar.rkb) tar.rkb = rkb wkb = long(eachLine.split()[2]) wpm = long(wkb - tar.wkb) tar.wkbpm = wpm tar.wkb = wkb tar.save() else: logger.warn( "Found target %s on %s that does not exist in the DB" % (iqntar, self.serverDNS)) def DeleteTarget(self, iqntar, vguuid): """ Delete target """ logger.info("Trying to delete target %s from VG %s on host %s" % (iqntar, vguuid, self.serverDNS)) if self.GetTargetsState() == -1: logger.error("Could not GetTargetsState while deleting %s" % (iqntar, )) return -1 try: tar = Target.objects.get(iqntar=iqntar) except: logger.warn("Could not find deletion target in DB, exiting. " + iqntar) return -1 if not tar.sessionup: cmdStr = " ".join([ "sudo", self.rembashpath, self.remoteinstallLoc + 'saturn-bashscripts/removetarget.sh', iqntar, vguuid ]) exStr = self.Exec(cmdStr) if exStr == -1: return -1 self.GitSave(vguuid, "Trying to delete target %s " % (iqntar, )) success1 = False success2 = False for eachLine in exStr: logger.info(eachLine) if "Removing virtual target '" + iqntar + "' from driver 'iscsi': done" in eachLine: success1 = True if "successfully removed" in eachLine: success2 = True if success1 == True and success2 == True: logger.info( "Successful deletion of target %s from VG %s on host %s" % (iqntar, vguuid, self.serverDNS)) return 1 else: logger.error("Error deleting target %s from VG %s on host %s" % (iqntar, vguuid, self.serverDNS)) return -1 return -1 def GetInterfaces(self): """ Scan and get network interfaces into saturnring DB """ #cmdStr = 'ifconfig | grep -oE "inet addr:[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}" | cut -d: -f2' cmdStr = 'ip addr | grep -oE "inet [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}" | cut -d" " -f2' ipadds = self.Exec(cmdStr) if ipadds == -1: return -1 sh = StorageHost.objects.get(dnsname=self.serverDNS) superuser = User.objects.filter( is_superuser=True).order_by('username')[0] for addr in ipadds: try: addr = addr.rstrip() if "127.0.0.1" in addr: #Ignore loopback addresses continue socket.inet_aton(addr) interfaces = Interface.objects.filter(ip=addr) if len(interfaces) != 1: #If 0, then new interface Interface.objects.filter(ip=addr).delete() logger.info( "Adding newly discovered interface %s to storage host %s " % (addr, self.serverDNS)) try: newInterface = Interface(storagehost=sh, ip=addr, owner=superuser) newInterface.save() for eachIPRange in IPRange.objects.all(): if ipaddress.ip_address( unicode(addr)) in ipaddress.ip_network( unicode(eachIPRange.iprange)): eachIPRange.hosts.add(sh) eachIPRange.save() newInterface.iprange.add(eachIPRange) newInterface.owner = eachIPRange.owner newInterface.save() except: logger.warn( "Error saving newly discovered Interface %s of host %s" % (addr, self.serverDNS)) var = format_exc() logger.warn(var) else: if interfaces[0].storagehost.dnsname != self.serverDNS: Interface.objects.filter(ip=addr).delete() logger.warn( "IP address %s was reassigned to another host" % (addr, )) except socket.error: logger.warn( "Invalid IP address %s retuned in GetInterfaces call on Saturn server %s " % (addr, self.serverDNS)) var = format_exc() logger.warn(var)
class SFTPCon: """ Establishes SSH connection to a remote server and helps in sending, moving and deleting files on the remote server. parameters host: str the hostname or IP of remote machine username: str your username at the remote machine password: str your password at the remote machine port: int the SSH port of the remote machine private_key: str path of the file containing the private SSH key private_key_pass: str password to use, if private_key is encrypted compression: boolean default True, use True to disable compression """ def __init__(self, host, username=None, password=None, port=22, private_key_file=None, private_key_password=None, compression=True): self._con = None # connection options cnopts = CnOpts() cnopts.compression = compression # set hostkeys to None, if not provided if private_key_file is None: cnopts.hostkeys = None if password is None: logging.debug('No password provided, using key auth.') # NOTE: # Ducking exceptions, so that they can be handled # by the main module however it wants. self._con = Connection(host=host, username=username, port=port, private_key_file=private_key_file, # Ignore LineLengthBear, PycodeStyleBear private_key_password=private_key_password, cnopts=cnopts) if self._con is None: self._con = Connection(host, username=username, port=port, password=password, cnopts=cnopts) def _send(self, src_path, remote_path): """ Transfers a resource(file) to the remote SFTP server. parameters src_path: str local path of the resource to be sent remote_path: str mapped path of the src_path on the remote storage """ # NOTE: For transferring any file, make sure all parent dir # exist, if not make them. # mkdir -p: no errors if existing, make parent dirs as needed parent_path = os.path.split(remote_path)[0] cmd = 'mkdir -p "' + parent_path + '"' self._con.execute(cmd) self._con.put(src_path, remote_path) def _delete(self, remote_path): """ Deletes a resource on the remote SFTP server. parameters remote_path: str path of the resource to be deleted on the remote storage """ # NOTE: Very dangerous cmd, can delete everything inside a dir # Use with CAUTION! cmd = 'rm -rf "' + remote_path + '"' self._con.execute(cmd) def _move(self, remote_src_path, remote_dest_path): """ Moves a resource on the remote SFTP server. parameters remote_src_path: str remote path of the resource before it was moved remote_dest_path: str remote path of the resource after it was moved """ # NOTE: For moving any file, make sure all parent dir # exist, if not make them. # mkdir -p: no errors if existing, make parent dirs as needed parent_path = os.path.split(remote_dest_path)[0] cmd = 'mkdir -p "' + parent_path + '"' self._con.execute(cmd) cmd = 'mv "' + remote_src_path + '" "' + remote_dest_path + '"' self._con.execute(cmd)
class PollServer(): """ This is the controller that calls/runs scripts on a Saturn server as required by saturnring """ def __init__(self,serverDNS): """ The init script for the class """ try: self.serverDNS = str(serverDNS) self.hostobject = StorageHost.objects.get(dnsname=self.serverDNS) BASE_DIR = dirname(dirname(__file__)) config = ConfigReader() self.userName = config.get('saturnnode','user') self.keyFile = join(BASE_DIR,config.get('saturnring','privatekeyfile')) self.rembashpath = config.get('saturnnode','bashpath') self.rempypath = config.get('saturnnode','pythonpath') self.iscsiconfdir = join(BASE_DIR,config.get('saturnring','iscsiconfigdir')) self.remoteinstallLoc = config.get('saturnnode','install_location') self.localbashscripts = join(BASE_DIR,config.get('saturnring','bashscripts')) except: logger.critical("Error setting up configuration for server "+self.serverDNS) logger.critical(format_exc()) try: self.srv = Connection(self.serverDNS,self.userName,self.keyFile) except: logger.critical("Failed SSH-exec connection on Saturn server %s; possible cause: %s" % (self.serverDNS,format_exc()) ) self.srv="inError" def CheckServer(self): if self.srv == 'inError': return -1 remotePath = join(self.remoteinstallLoc,'saturn-bashscripts') cmdStr = " ".join([join(remotePath,'checkserver.sh'), '2> checkservererror.log']) #logger.info("Executing %s on %s" %(cmdStr,self.serverDNS)) rtnStrList = self.Exec(cmdStr) if (rtnStrList == -1): return -2 else: for aLine in rtnStrList: if "FAILURE" in aLine: logger.error(self.serverDNS + ": "+ str(rtnStrList)) return -3 return 0 def InstallScripts(self): """ Copy bash scripts from the saturnringserver into the saturn server via sftp """ rtnVal = -1 try: if self.srv == "inError": raise Exception('Server SSH connection object inError') remotePath = join(self.remoteinstallLoc,'saturn-bashscripts') self.srv.execute (" ".join(['mkdir', '-p', remotePath])) self.srv.chdir(remotePath) locallist=listdir(self.localbashscripts) for localfile in locallist: self.srv.put(join(self.localbashscripts,localfile)) self.srv.execute(" ".join(["chmod", "777",join(remotePath,localfile)])) #Update rc.local for luks reboot functionality luksopenscriptpath = join(remotePath,'luksonreboot.sh'); self.srv.execute("sudo sed -i '/luksonreboot.sh/d' /etc/rc.local") #delete pre-existing line if any self.srv.execute("sudo sed -i '/^exit 0/i " + '/bin/bash ' + luksopenscriptpath +"' /etc/rc.local") logger.info("Installed scripts on "+ self.serverDNS) rtnVal = 1 except: logger.error('Could not install scripts on '+self.serverDNS) logger.error(format_exc()) finally: return rtnVal def Exec(self,command): """ Helper function for executing a remote command over an SSH tunnel """ rtncmd = -1 if self.srv=="inError": logger.error("There is no ssh connection object for server: %s" %(self.serverDNS,)) return -1 try: #srv = Connection(self.serverDNS,self.userName,self.keyFile) rtncmd=self.srv.execute(command) #srv.close() except: logger.error("Failed SSH-exec command: %s on Saturn server %s" % (command, self.serverDNS)) logger.error(format_exc()) return rtncmd def GetFile(self,remotePath,localPath): """ Get a file from the remote server. return 1 on success, -1 on error """ try: self.srv.get(remotePath,localPath) #logger.info("Copying file %s from remote server %s to local path %s succeeded" %(remotePath,self.serverDNS,localPath)) return 1 except: logger.error("Error copying file %s from remote server %s to local path %s" %(remotePath,self.serverDNS,localPath)) logger.error(format_exc()) return -1 def PutKeyFile(self,keyfileName): """ Copy over the keyfile to be used for creating the LUKs encrypted DM volumes """ remoteKeyfileDir = join(self.remoteinstallLoc,'keys') try: self.Exec (" ".join(['mkdir','-p',remoteKeyfileDir])) self.srv.chdir(remoteKeyfileDir) self.srv.put(join(self.iscsiconfdir,keyfileName)) self.remoteKeyfilePath = join(remoteKeyfileDir,keyfileName) rtnString = self.Exec ('test -f ' + self.remoteKeyfilePath + '&& echo "OK Putkeyfile" ') logger.info(rtnString) if "OK Putkeyfile" not in str(rtnString): raise ValueError("Putkey didnt install file") except ValueError: logger.error("Failed to put keyfile on Saturn server %s at location %s" %(self.serverDNS,join(remoteKeyfileDir,keyfileName))) logger.error(format_exc()) return -1 return self.remoteKeyfilePath def DelKeyFile(self,keyfileName): """ Delete key file from saturn server """ remoteKeyfileDir = join(self.remoteinstallLoc,'keys') self.srv.execute('rm '+ join(remoteKeyfileDir,keyfileName)) rtnString = self.Exec ('test ! -f ' + join(self.iscsiconfdir,keyfileName)+ ' && echo "OK Deleted keyfile"') return rtnString def ParseLVM(self,strList,delimitStr,paraList): """ Parse lvdisplay and vgdisplay strings and populate dictionaries with relevant information """ rtnDict ={} valueDict={} for aLine in strList: if (delimitStr in aLine): if len(valueDict) == len(paraList): rtnDict[valueDict[paraList[0]]]=valueDict valueDict = {} continue else: for anItem in paraList: if anItem in aLine: valueDict[anItem] = aLine.split(anItem)[1].strip() if '%' in valueDict[anItem]: valueDict[anItem] = float(valueDict[anItem][:-2]) continue if '/' in valueDict[anItem]: valueDict[anItem] = valueDict[anItem].split('/')[0] if (('GiB' in valueDict[anItem]) and ('Size' in aLine)): valueDict[anItem] = float(valueDict[anItem].split('GiB')[0])*1 continue if (('MiB' in valueDict[anItem]) and ('Size' in aLine)): valueDict[anItem] = float(valueDict[anItem].split('MiB')[0])*0.001 continue continue if len(valueDict) == len(paraList): rtnDict[valueDict[paraList[0]]] = valueDict #logger.info(rtnDict) return rtnDict def UpdateLVs(self,vgObject): """ Update LV information, called to monitor and update capacity. """ lvdict = self.GetLVs(vgObject.vguuid) if "No LVs " in lvdict: logger.info("There are no LVs in %s to run UpdateLVs on in Saturn host %s" %(vgObject.vguuid, self.serverDNS)) return 0 if lvdict == -1: logger.error ("Could not run GetLVs (perhaps there are no LVs in this VG yet?)") return -1 lvs = set(LV.objects.filter(vg=vgObject)) lvDict = {} for eachlv in lvs: lvDict[eachlv.lvname] = eachlv for lvName,lvinfo in lvdict.iteritems(): if lvName in lvDict: preexistLV=lvDict[lvName] preexistLV.lvsize=lvinfo['LV Size'] preexistLV.save(update_fields=['lvsize']) else: logger.warn("Found orphan LV %s in VG %s on host %s" %(lvName,vgObject.vguuid,self.serverDNS)) def GetLVs(self,vguuid): """ Wrapper for parselvm (for LVs), actually populating the DB is done by the UpdateLV function """ execCmd = " ".join(['sudo','vgdisplay', '-c','|','grep',vguuid,'|','cut -d: -f1']) vgname = self.Exec(execCmd)[0].strip() if vgname == -1: logger.error("Could not execute %s on %s " % (execCmd,self.serverDNS)) return -1 execCmd=" ".join(['sudo','lvdisplay','--units g',vgname]) lvStrList = self.Exec(execCmd) if lvStrList ==[""]: return "No LVs in %s" %(vguuid,) if lvStrList == -1: logger.error("Could not execute %s on %s " % (execCmd,self.serverDNS)) return -1 delimitStr = '--- Logical volume ---' paraList=['LV Name','LV UUID','LV Size'] lvs = self.ParseLVM(lvStrList,delimitStr,paraList) return lvs def GetVG(self): #Unit test this again """ Wrapper for parseLVM (for VGs)+populating the DB """ delimitStr = '--- Volume group ---' paraList = ['VG Name','VG Size','PE Size','Total PE', 'Free PE / Size', 'VG UUID'] execCmd = " ".join(['sudo','vgdisplay','--units g']) vgStrList = self.Exec(execCmd) if vgStrList == -1: logger.error("Error in GetVG while executing %s on server %s " %(execCmd,self.serverDNS)) return -1 vgs = self.ParseLVM(vgStrList,delimitStr,paraList) #logger.info("VGStating on %s returns %s " % (self.serverDNS, str(vgs)) ) rtnvguuidList = "" for vgname in vgs: try: execCmd = " ".join(['sudo',join(self.remoteinstallLoc,'saturn-bashscripts/vgstats.sh'),vgname,' 2> error.log']) cmdStr = self.Exec(execCmd) maxavl = float(cmdStr[0].rstrip()) totalGB = float(cmdStr[1].rstrip()) isThin = bool(int(cmdStr[2].rstrip())) except: logger.warn("Unable to run VGscan, disabling VG on "+self.serverDNS) logger.warn(format_exc()) try: vg = VG.objects.get(vguuid=vgs[vgname]['VG UUID']) vg.in_error = True vg.save(update_fields=['in_error']) except: logger.error("VG not found in DB: %s" % ( vgs[vgname]['VG UUID'],)) return 3 existingvgs = VG.objects.filter(vguuid=vgs[vgname]['VG UUID']) if len(existingvgs)==1: existingvg = existingvgs[0] existingvg.in_error=False existingvg.CurrentAllocGB = totalGB-maxavl#Target.objects.filter(targethost=existingvg.vghost).aggregate(Sum('sizeinGB'))['sizeinGB__sum'] existingvg.totalGB=totalGB existingvg.maxavlGB=maxavl existingvg.is_thin=isThin existingvg.vgsize = vgs[vgname]['VG Size'] existingvg.save(update_fields=['totalGB','maxavlGB','vgsize','CurrentAllocGB','in_error','is_thin']) #logger.info( "Ran in existingVG loop") else: logger.info("Found new VG, adding\n" + str(vgs[vgname])) myvg = VG(vghost=StorageHost.objects.get(dnsname=self.serverDNS),vgsize=vgs[vgname]['VG Size'], vguuid=vgs[vgname]['VG UUID'],vgpesize=vgs[vgname]['PE Size'], vgtotalpe=vgs[vgname]['Total PE'], vgfreepe=vgs[vgname]['Free PE / Size'], totalGB=totalGB,maxavlGB=maxavl, is_thin=isThin) myvg.save()#force_update=True) rtnvguuidList = rtnvguuidList+ ','+ vgs[vgname]['VG UUID'] return rtnvguuidList[1:] def GitSave(self,commentStr): """ Check in changes to config files into git repository """ try: repo = Repo(self.iscsiconfdir) filelist = [ f for f in listdir(self.iscsiconfdir) if isfile(join(self.iscsiconfdir,f)) ] repo.stage(filelist) repo.do_commit(commentStr) return 1 except: var = format_exc() logger.error("During GitSave %s: Git save error: %s" % (commentStr,var)) return -1 def CreateTarget(self,iqnTarget,iqnInit,sizeinGB,storageip1,storageip2,vguuid,isencrypted): """ Create iSCSI target by running the createtarget script; and save latest scst.conf from the remote server (overwrite) """ #self.srv = Connection(self.serverDNS,self.userName,self.keyFile) if str(isencrypted) != '1': cmdStr = " ".join(['sudo',self.rembashpath,join(self.remoteinstallLoc,'saturn-bashscripts','createtarget.sh'), str(sizeinGB),iqnTarget,storageip1,storageip2,iqnInit,vguuid, '2> createtarget.sh-error.log']) else: try: self.remotekeyfilelocation = self.PutKeyFile("cryptokey") cmdStr = " ".join(['sudo',self.rembashpath,join(self.remoteinstallLoc,'saturn-bashscripts','createencryptedtarget.sh'), str(sizeinGB),iqnTarget,storageip1,storageip2,iqnInit,vguuid,self.remotekeyfilelocation,'2> createencryptedtarget.sh-error.log']) if self.remotekeyfilelocation == -1: raise ValueError("Putkey failed") except: logger.error("Error setting up encrypted target: %s " %(iqnTarget,)) logger.error(format_exc()) return -1 #srv.close() logger.info ("Launching createtarget with \n%s" %(cmdStr,)) exStr=self.Exec(cmdStr) if exStr == -1: return -1 commentStr = "Trying to create target %s " %( iqnTarget, ) try: if self.GetFile('/temp/scst.conf',self.iscsiconfdir+self.serverDNS+'.scst.conf')==-1: raise Exception('Error getting scst.conf') if self.GetFile(join('/temp',vguuid),join(self.iscsiconfdir,self.serverDNS+'.'+vguuid+'.lvm'))==-1: raise Exception('Error getting LVM configuration file %s' %(vguuid+'.lvm',)) if self.GitSave(commentStr) == -1: raise Exception('Error in GitSave') except: logger.warning('Unable to save updated config files on ring server') logger.warning(format_exc()) logger.info("Execution report for %s: %s" %(cmdStr,"\t".join(exStr))) if "SUCCESS" in str(exStr): logger.info("Returning successful createtarget run") return 1 else: logger.error("Returning failed createtarget run:" + str(exStr)) return 0 def GetTargetsState(self): """ Read targets to determine their latest state via the parsetarget script """ cmdStr = " ".join(["sudo",self.rempypath, join(self.remoteinstallLoc,'saturn-bashscripts','parsetarget.py'), '2> parsetargeterror.txt']) exStr = self.Exec(cmdStr) #logger.info("Parse target returns " +str(exStr)) if exStr == -1: return -1 try: targetDic = {} hostTargets = set(Target.objects.filter(targethost=self.hostobject)) for eachTarget in hostTargets: targetDic[eachTarget.iqntar] = eachTarget for eachLine in exStr: iqntar = eachLine.split()[0] if iqntar in targetDic: tar = targetDic[iqntar] if "no session" in eachLine: tar.sessionup=False tar.rkbpm = 0 tar.wkbpm = 0 else: tar.sessionup=True rkb = long(eachLine.split()[1]) tar.rkbpm = long(rkb-tar.rkb) tar.rkb=rkb wkb = long(eachLine.split()[2]) wpm = long(wkb-tar.wkb) tar.wkbpm = wpm tar.wkb=wkb tar.save() else: logger.warn("Found target %s on %s that does not exist in the DB" % (iqntar,self.serverDNS) ) except: logger.error("Error reading iSCSI target state for %s on server %s" %(iqntar,self.serverDNS)) logger.error(format_exc()) return -1 return 0 def DeleteTarget(self,iqntar,vguuid,lvolname): """ Delete target """ logger.info("Trying to delete target %s from VG %s on host %s" %(iqntar,vguuid,self.serverDNS)) if self.GetTargetsState() == -1: logger.error("Could not GetTargetsState while deleting %s" %(iqntar,)) return -1 try: tar = Target.objects.get(iqntar=iqntar) except: logger.warn("Could not find deletion target in DB, exiting. "+iqntar) return -1 if not tar.sessionup: cmdStr = " ".join(["sudo",self.rembashpath,join(self.remoteinstallLoc,'saturn-bashscripts','removetarget.sh'),iqntar,vguuid,lvolname]) exStr = self.Exec(cmdStr) if exStr == -1: return -1 try: if self.GetFile('/temp/scst.conf',self.iscsiconfdir+self.serverDNS+'.scst.conf') == -1: raise Exception('Error getting scst configuration file to store locally') if self.GetFile('/temp/'+vguuid,self.iscsiconfdir+self.serverDNS+'.'+vguuid+'.lvm') == -1: raise Exception('Error getting LVM configuration to store locally') if self.GitSave("Trying to delete target "+iqntar) == -1: raise Exception('Error with gitsave in delete target') except: logger.error("Error getting configuration files after deletion of target") logger.error(format_exc()) success1 = False success2 = False logger.info(exStr) for eachLine in exStr: if "Removing virtual target '"+iqntar+"' from driver 'iscsi': done" in eachLine: success1=True if "successfully removed" in eachLine: success2=True if success1==True and success2==True: logger.info("Successful deletion of target %s from VG %s on host %s" %(iqntar,vguuid,self.serverDNS)) return 1 else: logger.error("Error deleting target %s from VG %s on host %s; command execution returned %s" %(iqntar,vguuid,self.serverDNS,str(exStr))) return -1 else: logger.error("Target state of %s is set to session up, will not try to delete it." %(iqntar,)) return -1 def GetInterfaces(self): """ Scan and get network interfaces into saturnring DB """ #cmdStr = 'ifconfig | grep -oE "inet addr:[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}" | cut -d: -f2' cmdStr = 'ip addr | grep -oE "inet [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}" | cut -d" " -f2 2> ipaddrerror.txt' ipadds=self.Exec(cmdStr) if ipadds == -1: return -1 rtnVal = 0 sh = self.hostobject superuser=User.objects.filter(is_superuser=True).order_by('username')[0] for addr in ipadds: try: addr = addr.rstrip() if "127.0.0.1" in addr: #Ignore loopback addresses continue socket.inet_aton(addr) interfaces = Interface.objects.filter(ip=addr) if len(interfaces) != 1: #If 0, then new interface Interface.objects.filter(ip=addr).delete() logger.info("Adding newly discovered interface %s to storage host %s " % (addr, self.serverDNS)) try: newInterface = Interface(storagehost=sh,ip=addr,owner=superuser) newInterface.save() for eachIPRange in IPRange.objects.all(): if ipaddress.ip_address(unicode(addr)) in ipaddress.ip_network(unicode(eachIPRange.iprange)): eachIPRange.hosts.add(sh) eachIPRange.save() newInterface.iprange.add(eachIPRange) newInterface.owner=eachIPRange.owner newInterface.save() except: logger.warn("Error saving newly discovered Interface %s of host %s" % (addr, self.serverDNS)) var = format_exc() logger.warn(var) rtnVal = -1 else: if interfaces[0].storagehost.dnsname != self.serverDNS: Interface.objects.filter(ip=addr).delete() logger.warn("IP address %s was reassigned to another host" % (addr,)) except socket.error: logger.warn("Invalid IP address %s retuned in GetInterfaces call on Saturn server %s " % (addr, self.serverDNS )) var = format_exc() logger.warn(var) rtnVal = -1 return rtnVal def InsertCrypttab(self,base_LV,enc_LV,keyfilePath): """ Insert entry into /etc/crypttab """ #Needed for the crypttab baselvpath = self.Exec(" ".join(["sudo sh -c 'lvs -o lv_path | grep ",base_LV," | tr -d \" \"'"]))[0].strip() if baselvpath == -1: return -1 logger.info("BaseLVPATH = " + baselvpath) cmdStr = " ".join(["sudo sh -c 'echo \"" + enc_LV, baselvpath, keyfilePath, "luks\" >> /etc/crypttab; mkdir -p /temp; cp /etc/crypttab /temp/crypttab; chmod 666 /temp/crypttab'"]) logger.info("InsertCrypttab: "+cmdStr) rtnVal = self.Exec(cmdStr) if rtnVal == -1: logger.error("Error in InsertCrypttab while executing %s" %(cmdStr,)) try: if self.GetFile('/temp/crypttab',self.iscsiconfdir+self.serverDNS+'.crypttab') == -1: raise Exception('Could not get crypttab file') if self.GitSave("Trying the insert crypttab entry " + cmdStr) == -1: raise Exception('Could not get gitsave to work for crypttab') except: logger.error('Error with getfile/gitsave during crypttab insert operations on %s' %(self.serverDNS,)) logger.error(format_exc()) def DeleteCrypttab(self,lvStr): """ Delete entry from /etc/crypttab LVStr can be either the encrypted LV name or the base LV name """ cmdStr = " ".join(['sudo sed -i','/'+lvStr+'/d','/etc/crypttab']) + "; sudo mkdir -p /temp; sudo cp /etc/crypttab /temp/crypttab; sudo chmod 666 /temp/crypttab" logger.info("DeleteCrypttab: "+cmdStr) rtnVal = self.Exec(cmdStr) if rtnVal == -1: return rtnVal logger.error("Error in DeleteCrypttab while executing %s" %(cmdStr,)) try: if self.GetFile('/temp/crypttab',self.iscsiconfdir+self.serverDNS+'.crypttab') == -1: raise Exception('Could not get crypttab file') if self.GitSave("Trying the insert crypttab entry " + cmdStr) == -1: raise Exception('Could not get gitsave to work for crypttab') except: logger.error('Error with getfile/gitsave during crypttab delete operations on %s' %(self.serverDNS,)) logger.error(format_exc())
class PollServer(): """ This is the controller that calls/runs scripts on a Saturn server as required by saturnring """ def __init__(self,serverDNS): """ The init script for the class """ self.serverDNS = str(serverDNS) BASE_DIR = dirname(dirname(__file__)) config = ConfigReader() self.userName = config.get('saturnnode','user') self.keyFile = join(BASE_DIR,config.get('saturnring','privatekeyfile')) self.rembashpath = config.get('saturnnode','bashpath') self.rempypath = config.get('saturnnode','pythonpath') self.iscsiconfdir = join(BASE_DIR,config.get('saturnring','iscsiconfigdir')) self.remoteinstallLoc = config.get('saturnnode','install_location') self.localbashscripts = join(BASE_DIR,config.get('saturnring','bashscripts')) try: self.srv = Connection(self.serverDNS,self.userName,self.keyFile) except: logger.critical("Failed SSH-exec connection on Saturn server %s; possible cause: %s" % (self.serverDNS,format_exc()) ) def InstallScripts(self): """ Copy bash scripts from the saturnringserver into the saturn server via sftp """ #srv = Connection(self.serverDNS,self.userName,self.keyFile) self.srv.execute ('mkdir -p '+self.remoteinstallLoc+'saturn-bashscripts/') self.srv.chdir(self.remoteinstallLoc+'saturn-bashscripts/') locallist=listdir(self.localbashscripts) for localfile in locallist: self.srv.put(self.localbashscripts+localfile) self.srv.execute("chmod 777 "+self.remoteinstallLoc+'saturn-bashscripts/'+localfile) #srv.close() logger.info("Installed scripts") def Exec(self,command): """ Helper function for executing a remote command over an SSH tunnel """ rtncmd = -1 try: #srv = Connection(self.serverDNS,self.userName,self.keyFile) rtncmd=self.srv.execute(command) #srv.close() except: logger.error("Failed SSH-exec command: %s on Saturn server %s" % (command, self.serverDNS)) logger.error(format_exc()) return rtncmd def ParseLVM(self,strList,delimitStr,paraList): """ Parse lvdisplay and vgdisplay strings and populate dictionaries with relevant information """ rtnDict ={} valueDict={} for aLine in strList: if (delimitStr in aLine): if len(valueDict) == len(paraList): rtnDict[valueDict[paraList[0]]]=valueDict valueDict = {} continue else: for anItem in paraList: if anItem in aLine: valueDict[anItem] = aLine.split(anItem)[1].strip() if '%' in valueDict[anItem]: valueDict[anItem] = float(valueDict[anItem][:-2]) continue if '/' in valueDict[anItem]: valueDict[anItem] = valueDict[anItem].split('/')[0] if 'GiB' in valueDict[anItem]: valueDict[anItem] = float(valueDict[anItem].split('GiB')[0])*1 continue if 'MiB' in valueDict[anItem]: valueDict[anItem] = float(valueDict[anItem].split('MiB')[0])*0.001 continue continue if len(valueDict) == len(paraList): rtnDict[valueDict[paraList[0]]] = valueDict logger.info(rtnDict) return rtnDict def UpdateLVs(self,vgObject): """ Update LV information, called to monitor and update capacity. """ lvdict = self.GetLVs(vgObject.vguuid) if "No LVs " in lvdict: logger.info("There are no LVs in %s to run UpdateLVs on in Saturn host %s" %(vgObject.vguuid, self.serverDNS)) return 0 if lvdict == -1: logger.error ("Could not run GetLVs (perhaps there are no LVs in this VG yet?)") return -1 lvs = LV.objects.filter(vg=vgObject) for lvName,lvinfo in lvdict.iteritems(): if len(lvs.filter(lvname=lvName)): preexistLV=lvs.filter(lvname=lvName)[0] preexistLV.lvsize=lvinfo['LV Size'] preexistLV.save(update_fields=['lvsize']) else: logger.warn("Found orphan LV %s in VG %s on host %s" %(lvName,vgObject.vguuid,self.serverDNS)) def GetLVs(self,vguuid): """ Wrapper for parselvm (for LVs), actually populating the DB is done by the UpdateLV function """ execCmd = " ".join(['sudo','vgdisplay', '-c','|','grep',vguuid,'|','cut -d: -f1']) vgname = self.Exec(execCmd)[0].strip() if vgname == -1: logger.error("Could not execute %s on %s " % (execCmd,self.serverDNS)) return -1 execCmd=" ".join(['sudo','lvdisplay','--units g',vgname]) lvStrList = self.Exec(execCmd) if lvStrList ==[""]: return "No LVs in %s" %(vguuid,) if lvStrList == -1: logger.error("Could not execute %s on %s " % (execCmd,self.serverDNS)) return -1 delimitStr = '--- Logical volume ---' paraList=['LV Name','LV UUID','LV Size'] lvs = self.ParseLVM(lvStrList,delimitStr,paraList) return lvs def GetVG(self): #Unit test this again """ Wrapper for parseLVM (for VGs)+populating the DB """ delimitStr = '--- Volume group ---' paraList = ['VG Name','VG Size','PE Size','Total PE', 'Free PE / Size', 'VG UUID'] execCmd = " ".join(['sudo','vgdisplay','--units g']) vgStrList = self.Exec(execCmd) if vgStrList == -1: return -1 vgs = self.ParseLVM(vgStrList,delimitStr,paraList) #logger.info("VGStating on %s returns %s " % (self.serverDNS, str(vgs)) ) rtnvguuidList = "" for vgname in vgs: try: execCmd = " ".join(['sudo',self.remoteinstallLoc+'saturn-bashscripts/vgstats.sh',vgname]) cmdStr = self.Exec(execCmd) logger.info(self.serverDNS+": "+" ".join(['sudo',self.rembashpath,self.remoteinstallLoc+'saturn-bashscripts/vgstats.sh',vgname])+': returned '+str(cmdStr)) maxavl = float(cmdStr[0].rstrip()) totalGB = float(cmdStr[1].rstrip()) isThin = bool(int(cmdStr[2].rstrip())) except: logger.warn("Unable to run VGscan, disabling VG on "+self.serverDNS) logger.warn(format_exc()) try: vg = VG.objects.get(vguuid=vgs[vgname]['VG UUID']) vg.in_error = True vg.save(update_fields=['in_error']) except: logger.error("VG not found in DB: %s" % ( vgs[vgname]['VG UUID'],)) return 3 existingvgs = VG.objects.filter(vguuid=vgs[vgname]['VG UUID']) if len(existingvgs)==1: existingvg = existingvgs[0] existingvg.in_error=False existingvg.CurrentAllocGB = totalGB-maxavl#Target.objects.filter(targethost=existingvg.vghost).aggregate(Sum('sizeinGB'))['sizeinGB__sum'] existingvg.totalGB=totalGB existingvg.maxavlGB=maxavl existingvg.is_thin=isThin existingvg.vgsize = vgs[vgname]['VG Size'] existingvg.save(update_fields=['totalGB','maxavlGB','vgsize','CurrentAllocGB','in_error','is_thin']) logger.info( "Ran in existingVG loop") else: logger.info("Found new VG, adding\n" + str(vgs[vgname])) myvg = VG(vghost=StorageHost.objects.get(dnsname=self.serverDNS),vgsize=vgs[vgname]['VG Size'], vguuid=vgs[vgname]['VG UUID'],vgpesize=vgs[vgname]['PE Size'], vgtotalpe=vgs[vgname]['Total PE'], vgfreepe=vgs[vgname]['Free PE / Size'], totalGB=totalGB,maxavlGB=maxavl, is_thin=isThin) myvg.save()#force_update=True) rtnvguuidList = rtnvguuidList+ ','+ vgs[vgname]['VG UUID'] return rtnvguuidList[1:] def GitSave(self,vguuid,commentStr): """ Check in changes to config files into git repository """ try: #srv = Connection(self.serverDNS,self.userName,self.keyFile) self.srv.get('/temp/scst.conf',self.iscsiconfdir+self.serverDNS+'.scst.conf') self.srv.get('/temp/'+vguuid,self.iscsiconfdir+self.serverDNS+'.'+vguuid+'.lvm') try: repo = Repo(self.iscsiconfdir) filelist = [ f for f in listdir(self.iscsiconfdir) if isfile(join(self.iscsiconfdir,f)) ] repo.stage(filelist) repo.do_commit(commentStr) except: var = format_exc() logger.error("During GitSave: %s: Git save error: %s" % (commentStr, var)) except: var = format_exc() logger.error("During GitSave: %s: PYSFTP download error: %s" % (commentStr, var)) def CreateTarget(self,iqnTarget,iqnInit,sizeinGB,storageip1,storageip2,vguuid): """ Create iSCSI target by running the createtarget script; and save latest scst.conf from the remote server (overwrite) """ #self.srv = Connection(self.serverDNS,self.userName,self.keyFile) cmdStr = " ".join(['sudo',self.rembashpath,self.remoteinstallLoc+'saturn-bashscripts/createtarget.sh',str(sizeinGB),iqnTarget,storageip1,storageip2,iqnInit,vguuid]) #srv.close() logger.info ("Launching createtarget with \n%s" %(cmdStr,)) exStr=self.Exec(cmdStr) if exStr == -1: return -1 commentStr = "Trying to create target %s " %( iqnTarget, ) self.GitSave(vguuid,commentStr) logger.info("Execution report for %s: %s" %(cmdStr,"\t".join(exStr))) if "SUCCESS" in str(exStr): logger.info("Returning successful createtarget run") return 1 else: logger.error("Returning failed createtarget run") return 0 def GetTargetsState(self): """ Read targets to determine their latest state via the parsetarget script """ cmdStr = " ".join(["sudo",self.rempypath,self.remoteinstallLoc+'saturn-bashscripts/parsetarget.py']) exStr = self.Exec(cmdStr) if exStr == -1: return -1 for eachLine in exStr: iqntar = eachLine.split()[0] tar = Target.objects.filter(iqntar=iqntar) if len(tar)==1: tar = tar[0] if "no session" in eachLine: tar.sessionup=False tar.rkbpm = 0 tar.wkbpm = 0 else: tar.sessionup=True rkb = long(eachLine.split()[1]) tar.rkbpm = long(rkb-tar.rkb) tar.rkb=rkb wkb = long(eachLine.split()[2]) wpm = long(wkb-tar.wkb) tar.wkbpm = wpm tar.wkb=wkb tar.save() else: logger.warn("Found target %s on %s that does not exist in the DB" % (iqntar,self.serverDNS) ) def DeleteTarget(self,iqntar,vguuid): """ Delete target """ logger.info("Trying to delete target %s from VG %s on host %s" %(iqntar,vguuid,self.serverDNS)) if self.GetTargetsState() == -1: logger.error("Could not GetTargetsState while deleting %s" %(iqntar,)) return -1 try: tar = Target.objects.get(iqntar=iqntar) except: logger.warn("Could not find deletion target in DB, exiting. "+iqntar) return -1 if not tar.sessionup: cmdStr = " ".join(["sudo",self.rembashpath,self.remoteinstallLoc+'saturn-bashscripts/removetarget.sh',iqntar,vguuid]) exStr = self.Exec(cmdStr) if exStr == -1: return -1 self.GitSave(vguuid,"Trying to delete target %s " %( iqntar,)) success1 = False success2 = False for eachLine in exStr: logger.info(eachLine) if "Removing virtual target '"+iqntar+"' from driver 'iscsi': done" in eachLine: success1=True if "successfully removed" in eachLine: success2=True if success1==True and success2==True: logger.info("Successful deletion of target %s from VG %s on host %s" %(iqntar,vguuid,self.serverDNS)) return 1 else: logger.error("Error deleting target %s from VG %s on host %s" %(iqntar,vguuid,self.serverDNS)) return -1 return -1 def GetInterfaces(self): """ Scan and get network interfaces into saturnring DB """ #cmdStr = 'ifconfig | grep -oE "inet addr:[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}" | cut -d: -f2' cmdStr = 'ip addr | grep -oE "inet [0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}" | cut -d" " -f2' ipadds=self.Exec(cmdStr) if ipadds == -1: return -1 sh = StorageHost.objects.get(dnsname=self.serverDNS) superuser=User.objects.filter(is_superuser=True).order_by('username')[0] for addr in ipadds: try: addr = addr.rstrip() if "127.0.0.1" in addr: #Ignore loopback addresses continue socket.inet_aton(addr) interfaces = Interface.objects.filter(ip=addr) if len(interfaces) != 1: #If 0, then new interface Interface.objects.filter(ip=addr).delete() logger.info("Adding newly discovered interface %s to storage host %s " % (addr, self.serverDNS)) try: newInterface = Interface(storagehost=sh,ip=addr,owner=superuser) newInterface.save() for eachIPRange in IPRange.objects.all(): if ipaddress.ip_address(unicode(addr)) in ipaddress.ip_network(unicode(eachIPRange.iprange)): eachIPRange.hosts.add(sh) eachIPRange.save() newInterface.iprange.add(eachIPRange) newInterface.owner=eachIPRange.owner newInterface.save() except: logger.warn("Error saving newly discovered Interface %s of host %s" % (addr, self.serverDNS)) var = format_exc() logger.warn(var) else: if interfaces[0].storagehost.dnsname != self.serverDNS: Interface.objects.filter(ip=addr).delete() logger.warn("IP address %s was reassigned to another host" % (addr,)) except socket.error: logger.warn("Invalid IP address %s retuned in GetInterfaces call on Saturn server %s " % (addr, self.serverDNS )) var = format_exc() logger.warn(var)