class SFtpHandler(object):
def __init__(self, logger=None):
self.logger = logger or MyLogger()
self.paramiko = __import__('paramiko')
self.stat = __import__('stat')
self.os = __import__('os')
self.BackupUtils = BackupUtils()
"""
Sftp Directory transversal
"""
def sftpWalk(self, sftp, remotepath, remote_dir, backup_folder_mirror, files=[], folders=[]):
for f in sftp.listdir_attr(remotepath):
fullFilePath = remotepath + "/" + f.filename
localFilePath = fullFilePath.replace(remote_dir,
backup_folder_mirror)
if self.stat.S_ISDIR(f.st_mode):
if f.filename != ".svn":
self.BackupUtils.createDirectory(localFilePath)
folders.append(fullFilePath)
self.sftpWalk(sftp, fullFilePath, remote_dir,
backup_folder_mirror, files, folders)
else:
files.append(fullFilePath)
self.logger.doLog("Retrieving file from %s to %s" %
(fullFilePath, localFilePath))
sftp.get(fullFilePath, localFilePath)
return (remotepath, folders, files)
def __init__(self, user, password, host, port, remote_dir, backup_folder_mirror, backup_folder_diff, backup_folder_archive, logger = None):
self.logger = logger or MyLogger()
self.user = user
self.password = password
self.host = host
self.port = port
self.remote_dir = remote_dir
self.backup_folder_mirror = backup_folder_mirror
self.backup_folder_diff = backup_folder_diff
self.backup_folder_archive = backup_folder_archive
self.paramiko = None
self.stat = None
self.BackupUtils = BackupUtils()
self.paramiko = __import__('paramiko')
self.sftpHandler = SFtpHandler(logger)
class SshHandler(object):
def __init__(self, user, password, host, port, remote_dir, backup_folder_mirror, backup_folder_diff, backup_folder_archive, logger = None):
self.logger = logger or MyLogger()
self.user = user
self.password = password
self.host = host
self.port = port
self.remote_dir = remote_dir
self.backup_folder_mirror = backup_folder_mirror
self.backup_folder_diff = backup_folder_diff
self.backup_folder_archive = backup_folder_archive
self.paramiko = None
self.stat = None
self.BackupUtils = BackupUtils()
self.paramiko = __import__('paramiko')
self.sftpHandler = SFtpHandler(logger)
#self.pexpect = __import__('pexpect')
"""
Basic function to be called to do mirroring
"""
def doMirror(self):
s = self.paramiko.SSHClient()
s.set_missing_host_key_policy(self.paramiko.AutoAddPolicy())
s.connect(self.host,self.port,username=self.user,password=self.password,timeout=4)
sftp = s.open_sftp()
self.sftpHandler.sftpWalk(sftp, self.remote_dir, self.remote_dir, self.backup_folder_mirror)
sftp.close()
s.close()
def doDiff(self):
self.BackupUtils.doRdiffBackupCall(self.backup_folder_mirror, self.backup_folder_diff)
def doBackup(self):
# Generate monthly archive name
current_year, current_month = self.BackupUtils.returnBasicDateInfo()
monthly_archive = self.retriveMonthlyArchive(current_year, current_month)
print monthly_archive
snapshot_date = "%04d-%02d-01" % (current_year, current_month)
self.BackupUtils.doBackup(monthly_archive, snapshot_date, self.backup_folder_diff, self.backup_folder_archive)
def retriveMonthlyArchive(self, currentYear, currentMonth):
return self.BackupUtils.returnFormatedMonthlyArchive(self.backup_folder_archive, currentYear, currentMonth)
def __init__(self, user, password, host, port, remote_dir, is_ftps, backup_folder_mirror, backup_folder_diff, backup_folder_archive, logger = None):
self.logger = logger or MyLogger()
self.user = user
self.password = password
self.host = host
self.port = port
self.remote_dir = remote_dir
self.is_ftps = is_ftps
self.backup_folder_mirror = backup_folder_mirror
self.backup_folder_diff = backup_folder_diff
self.backup_folder_archive = backup_folder_archive
self.urllib = __import__('urllib')
self.paramiko = None
self.stat = None
self.BackupUtils = BackupUtils()
self.sftpHandler = None
self.ftputil = None
if self.is_ftps:
self.paramiko = __import__('paramiko')
self.stat = __import__('stat')
self.os = __import__('os')
self.sftpHandler = SFtpHandler(logger)
else:
self.ftputil = __import__('ftputil')
def __init__(self, logger=None):
self.logger = logger or MyLogger()
self.paramiko = __import__('paramiko')
self.stat = __import__('stat')
self.os = __import__('os')
self.BackupUtils = BackupUtils()
# , 'local_dir' : 'mysqldump+ssh-test'
# },
# { 'title' : 'Direct MySQL dump with exotic MySQL server port'
# , 'type' : 'mysqldump'
# , 'db_user' : 'root'
# , 'db_pass' : '<mysql-password>'
# , 'db_host' : 'sql.machine.com'
# , 'db_port' : 9999
# , 'local_dir' : 'mysqldump-test'
# },
]
if __name__ == "__main__":
logger = MyLogger()
logger.doLog("starting!!")
backupUtils = BackupUtils(logger)
backupUtils.checkPythonVersion()
backupUtils.checkOperativeSystem()
backupUtils.checkRootDirectory(root_backup_dir)
backupUtils.checkCommands(["rdiff-backup", "rm", "tar", "bzip2"])
backupObjectsList = []
# Doing this right now is nicer to the user: thanks to this he doesn't need to wait the end of the (X)th backup to get the error about the (X+1)th
for backup in backup_list:
# Check datas and requirement for each backup
backup = backupUtils.parseBackupDefinition(backup)
backup_folders = backupUtils.returnDirectoryList(root_backup_dir, backup["local_dir"])
backupUtils.checkDirectories(root_backup_dir, backup["local_dir"])
if backup["type"] == "":
continue
if backup["type"] in ["FTP", "FTPS"]:
is_ftps = False
class FtpHandler(object):
def __init__(self, user, password, host, port, remote_dir, is_ftps, backup_folder_mirror, backup_folder_diff, backup_folder_archive, logger = None):
self.logger = logger or MyLogger()
self.user = user
self.password = password
self.host = host
self.port = port
self.remote_dir = remote_dir
self.is_ftps = is_ftps
self.backup_folder_mirror = backup_folder_mirror
self.backup_folder_diff = backup_folder_diff
self.backup_folder_archive = backup_folder_archive
self.urllib = __import__('urllib')
self.paramiko = None
self.stat = None
self.BackupUtils = BackupUtils()
self.sftpHandler = None
self.ftputil = None
if self.is_ftps:
self.paramiko = __import__('paramiko')
self.stat = __import__('stat')
self.os = __import__('os')
self.sftpHandler = SFtpHandler(logger)
else:
self.ftputil = __import__('ftputil')
"""
Basic function to be called to do mirroring
"""
def doMirror(self):
remote_url = "ftp://%s:%s@%s:%s/%s" % ( self.urllib.quote_plus(self.user)
, self.urllib.quote_plus(self.password)
, self.urllib.quote_plus(self.host)
, self.port
, self.urllib.quote(self.remote_dir)
)
# Force SSL layer for secure FTP
secure_options = ''
if self.is_ftps:
secure_options = 'set ftp:ssl-force true && set ftp:ssl-protect-data true && '
# Get a copy of the remote directory
ftp_backup = """lftp -c '%sset ftp:list-options -a && open -e "mirror -e --verbose=3 --parallel=2 . %s" %s'""" % (secure_options, self.backup_folder_mirror, remote_url)
self.logger.doLog("Going to mirror with command: %s " % ftp_backup)
#print self.getstatusoutput(ftp_backup)
if self.is_ftps:
#do ftps things
self.doMirrorSFtp()
else:
#do non ftps things
self.doMirrorFtp()
#finishing do mirror
"""
Actual Ftp mirroring
"""
def doMirrorFtp(self):
if int(self.port) != 21:
self.logger.doLog("For now only port 21")
else:
with self.ftputil.FTPHost(self.host, self.user, self.password) as host:
print host.curdir
for root, dirs, files in host.walk(self.remote_dir):
localDirectoryRoot = root.replace(self.remote_dir, self.backup_folder_mirror)
for directory in dirs:
if directory != ".svn":
self.BackupUtils.createDirectory(localDirectoryRoot + "/" + directory)
for remoteFile in files:
if remoteFile not in ['.ftpquota', '.DS_Store']:
if not remoteFile.startswith("log-"):
try:
self.logger.doLog("Retrieving file from %s to %s" % (root + "/" + remoteFile, localDirectoryRoot + "/" + remoteFile))
host.download(root + "/" + remoteFile, localDirectoryRoot + "/" + remoteFile)
except Exception as fileE:
print fileE
else:
self.logger.doLog("Log file %s will not be downloaded" % remoteFile)
print " =============================== "
"""
Actual SFtp mirroring
"""
def doMirrorSFtp(self):
#doing stuff
transport = self.paramiko.Transport((self.host, int(self.port)))
transport.connect(username = self.user, password = self.password)
sftp = self.paramiko.SFTPClient.from_transport(transport)
print self.remote_dir
#print sftp.listdir(self.remote_dir)
self.sftpHandler.sftpWalk(sftp, self.remote_dir, self.remote_dir, self.backup_folder_mirror)
#Do full backup
sftp.close()
transport.close()
def doDiff(self):
self.BackupUtils.doRdiffBackupCall(self.backup_folder_mirror, self.backup_folder_diff)
def retriveMonthlyArchive(self, currentYear, currentMonth):
return self.BackupUtils.returnFormatedMonthlyArchive(self.backup_folder_archive, currentYear, currentMonth)
def doBackup(self):
# Generate monthly archive name
current_year, current_month = self.BackupUtils.returnBasicDateInfo()
monthly_archive = self.retriveMonthlyArchive(current_year, current_month)
print monthly_archive
snapshot_date = "%04d-%02d-01" % (current_year, current_month)
self.BackupUtils.doBackup(monthly_archive, snapshot_date, self.backup_folder_diff, self.backup_folder_archive)
