class SFtpHandler(object): def __init__(self, logger=None): self.logger = logger or MyLogger() self.paramiko = __import__('paramiko') self.stat = __import__('stat') self.os = __import__('os') self.BackupUtils = BackupUtils() """ Sftp Directory transversal """ def sftpWalk(self, sftp, remotepath, remote_dir, backup_folder_mirror, files=[], folders=[]): for f in sftp.listdir_attr(remotepath): fullFilePath = remotepath + "/" + f.filename localFilePath = fullFilePath.replace(remote_dir, backup_folder_mirror) if self.stat.S_ISDIR(f.st_mode): if f.filename != ".svn": self.BackupUtils.createDirectory(localFilePath) folders.append(fullFilePath) self.sftpWalk(sftp, fullFilePath, remote_dir, backup_folder_mirror, files, folders) else: files.append(fullFilePath) self.logger.doLog("Retrieving file from %s to %s" % (fullFilePath, localFilePath)) sftp.get(fullFilePath, localFilePath) return (remotepath, folders, files)
def __init__(self, user, password, host, port, remote_dir, backup_folder_mirror, backup_folder_diff, backup_folder_archive, logger = None): self.logger = logger or MyLogger() self.user = user self.password = password self.host = host self.port = port self.remote_dir = remote_dir self.backup_folder_mirror = backup_folder_mirror self.backup_folder_diff = backup_folder_diff self.backup_folder_archive = backup_folder_archive self.paramiko = None self.stat = None self.BackupUtils = BackupUtils() self.paramiko = __import__('paramiko') self.sftpHandler = SFtpHandler(logger)
class SshHandler(object): def __init__(self, user, password, host, port, remote_dir, backup_folder_mirror, backup_folder_diff, backup_folder_archive, logger = None): self.logger = logger or MyLogger() self.user = user self.password = password self.host = host self.port = port self.remote_dir = remote_dir self.backup_folder_mirror = backup_folder_mirror self.backup_folder_diff = backup_folder_diff self.backup_folder_archive = backup_folder_archive self.paramiko = None self.stat = None self.BackupUtils = BackupUtils() self.paramiko = __import__('paramiko') self.sftpHandler = SFtpHandler(logger) #self.pexpect = __import__('pexpect') """ Basic function to be called to do mirroring """ def doMirror(self): s = self.paramiko.SSHClient() s.set_missing_host_key_policy(self.paramiko.AutoAddPolicy()) s.connect(self.host,self.port,username=self.user,password=self.password,timeout=4) sftp = s.open_sftp() self.sftpHandler.sftpWalk(sftp, self.remote_dir, self.remote_dir, self.backup_folder_mirror) sftp.close() s.close() def doDiff(self): self.BackupUtils.doRdiffBackupCall(self.backup_folder_mirror, self.backup_folder_diff) def doBackup(self): # Generate monthly archive name current_year, current_month = self.BackupUtils.returnBasicDateInfo() monthly_archive = self.retriveMonthlyArchive(current_year, current_month) print monthly_archive snapshot_date = "%04d-%02d-01" % (current_year, current_month) self.BackupUtils.doBackup(monthly_archive, snapshot_date, self.backup_folder_diff, self.backup_folder_archive) def retriveMonthlyArchive(self, currentYear, currentMonth): return self.BackupUtils.returnFormatedMonthlyArchive(self.backup_folder_archive, currentYear, currentMonth)
def __init__(self, user, password, host, port, remote_dir, is_ftps, backup_folder_mirror, backup_folder_diff, backup_folder_archive, logger = None): self.logger = logger or MyLogger() self.user = user self.password = password self.host = host self.port = port self.remote_dir = remote_dir self.is_ftps = is_ftps self.backup_folder_mirror = backup_folder_mirror self.backup_folder_diff = backup_folder_diff self.backup_folder_archive = backup_folder_archive self.urllib = __import__('urllib') self.paramiko = None self.stat = None self.BackupUtils = BackupUtils() self.sftpHandler = None self.ftputil = None if self.is_ftps: self.paramiko = __import__('paramiko') self.stat = __import__('stat') self.os = __import__('os') self.sftpHandler = SFtpHandler(logger) else: self.ftputil = __import__('ftputil')
def __init__(self, logger=None): self.logger = logger or MyLogger() self.paramiko = __import__('paramiko') self.stat = __import__('stat') self.os = __import__('os') self.BackupUtils = BackupUtils()
# , 'local_dir' : 'mysqldump+ssh-test' # }, # { 'title' : 'Direct MySQL dump with exotic MySQL server port' # , 'type' : 'mysqldump' # , 'db_user' : 'root' # , 'db_pass' : '<mysql-password>' # , 'db_host' : 'sql.machine.com' # , 'db_port' : 9999 # , 'local_dir' : 'mysqldump-test' # }, ] if __name__ == "__main__": logger = MyLogger() logger.doLog("starting!!") backupUtils = BackupUtils(logger) backupUtils.checkPythonVersion() backupUtils.checkOperativeSystem() backupUtils.checkRootDirectory(root_backup_dir) backupUtils.checkCommands(["rdiff-backup", "rm", "tar", "bzip2"]) backupObjectsList = [] # Doing this right now is nicer to the user: thanks to this he doesn't need to wait the end of the (X)th backup to get the error about the (X+1)th for backup in backup_list: # Check datas and requirement for each backup backup = backupUtils.parseBackupDefinition(backup) backup_folders = backupUtils.returnDirectoryList(root_backup_dir, backup["local_dir"]) backupUtils.checkDirectories(root_backup_dir, backup["local_dir"]) if backup["type"] == "": continue if backup["type"] in ["FTP", "FTPS"]: is_ftps = False
class FtpHandler(object): def __init__(self, user, password, host, port, remote_dir, is_ftps, backup_folder_mirror, backup_folder_diff, backup_folder_archive, logger = None): self.logger = logger or MyLogger() self.user = user self.password = password self.host = host self.port = port self.remote_dir = remote_dir self.is_ftps = is_ftps self.backup_folder_mirror = backup_folder_mirror self.backup_folder_diff = backup_folder_diff self.backup_folder_archive = backup_folder_archive self.urllib = __import__('urllib') self.paramiko = None self.stat = None self.BackupUtils = BackupUtils() self.sftpHandler = None self.ftputil = None if self.is_ftps: self.paramiko = __import__('paramiko') self.stat = __import__('stat') self.os = __import__('os') self.sftpHandler = SFtpHandler(logger) else: self.ftputil = __import__('ftputil') """ Basic function to be called to do mirroring """ def doMirror(self): remote_url = "ftp://%s:%s@%s:%s/%s" % ( self.urllib.quote_plus(self.user) , self.urllib.quote_plus(self.password) , self.urllib.quote_plus(self.host) , self.port , self.urllib.quote(self.remote_dir) ) # Force SSL layer for secure FTP secure_options = '' if self.is_ftps: secure_options = 'set ftp:ssl-force true && set ftp:ssl-protect-data true && ' # Get a copy of the remote directory ftp_backup = """lftp -c '%sset ftp:list-options -a && open -e "mirror -e --verbose=3 --parallel=2 . %s" %s'""" % (secure_options, self.backup_folder_mirror, remote_url) self.logger.doLog("Going to mirror with command: %s " % ftp_backup) #print self.getstatusoutput(ftp_backup) if self.is_ftps: #do ftps things self.doMirrorSFtp() else: #do non ftps things self.doMirrorFtp() #finishing do mirror """ Actual Ftp mirroring """ def doMirrorFtp(self): if int(self.port) != 21: self.logger.doLog("For now only port 21") else: with self.ftputil.FTPHost(self.host, self.user, self.password) as host: print host.curdir for root, dirs, files in host.walk(self.remote_dir): localDirectoryRoot = root.replace(self.remote_dir, self.backup_folder_mirror) for directory in dirs: if directory != ".svn": self.BackupUtils.createDirectory(localDirectoryRoot + "/" + directory) for remoteFile in files: if remoteFile not in ['.ftpquota', '.DS_Store']: if not remoteFile.startswith("log-"): try: self.logger.doLog("Retrieving file from %s to %s" % (root + "/" + remoteFile, localDirectoryRoot + "/" + remoteFile)) host.download(root + "/" + remoteFile, localDirectoryRoot + "/" + remoteFile) except Exception as fileE: print fileE else: self.logger.doLog("Log file %s will not be downloaded" % remoteFile) print " =============================== " """ Actual SFtp mirroring """ def doMirrorSFtp(self): #doing stuff transport = self.paramiko.Transport((self.host, int(self.port))) transport.connect(username = self.user, password = self.password) sftp = self.paramiko.SFTPClient.from_transport(transport) print self.remote_dir #print sftp.listdir(self.remote_dir) self.sftpHandler.sftpWalk(sftp, self.remote_dir, self.remote_dir, self.backup_folder_mirror) #Do full backup sftp.close() transport.close() def doDiff(self): self.BackupUtils.doRdiffBackupCall(self.backup_folder_mirror, self.backup_folder_diff) def retriveMonthlyArchive(self, currentYear, currentMonth): return self.BackupUtils.returnFormatedMonthlyArchive(self.backup_folder_archive, currentYear, currentMonth) def doBackup(self): # Generate monthly archive name current_year, current_month = self.BackupUtils.returnBasicDateInfo() monthly_archive = self.retriveMonthlyArchive(current_year, current_month) print monthly_archive snapshot_date = "%04d-%02d-01" % (current_year, current_month) self.BackupUtils.doBackup(monthly_archive, snapshot_date, self.backup_folder_diff, self.backup_folder_archive)