def syncSnapshotContentToS3SyncDirAndCreateSnapshot(nodeS3Path,keyspace,snapshotDate,s3SnapshotFolderName): # Get Current working directory scriptExecutionPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory snapshotDirColumnFamilyPaths = getLocalColumnFamilyPathsToSync(keyspace,snapshotDate) syncDir = 'sync_dir' action = 'backup' # It's important for deciding sync or snapshots directory for uploading files s3SnapshotFolderName = datetime.datetime.strftime(datetime.datetime.now(), '%Y%m%d') for snapshotDirColumnFamilyPath in snapshotDirColumnFamilyPaths: columnFamily=getColumnFamilyName(snapshotDirColumnFamilyPath) s3SyncDir = nodeS3Path + "/sync_dir/"+keyspace + "/"+columnFamily os.chdir(snapshotDirColumnFamilyPath) for newFile in getNewlyAddedFiles(snapshotDirColumnFamilyPath,nodeS3Path,syncDir,keyspace,s3SnapshotFolderName,columnFamily,action): copyFileToS3(newFile,s3SyncDir) for toBeRemovedFile in getListOfDeletedFiles(snapshotDirColumnFamilyPath,nodeS3Path,syncDir,keyspace,columnFamily,action,s3SnapshotFolderName): deleteFilesFromS3(toBeRemovedFile,s3SyncDir) os.chdir(scriptExecutionPath) createSnapshotAtS3(nodeS3Path, keyspace, snapshotDate,columnFamily,s3SyncDir)
def syncIncrementalBackupIntoLastSnapshotFolder(): validateKeyspace() createIncrementalBackup() nodeS3Path = getNodeBackupS3Path() latestAvailableSnapshotFolder = getLatestAvailableSnapshotFolderNameAtS3() syncDir = 'snapshots' action = 'incremental' # It's important for deciding sync or snapshots directory for uploading files for incrementalColumnFamilyPath in getIncrementalBackupDirectoriesPath(): keyspacePath=incrementalColumnFamilyPath.split("/backups")[0] keyspaceAndColumnFamilies=keyspacePath.split(cassandraDataDir())[1] columnFamily=keyspaceAndColumnFamilies.split("/")[2] incrementalBackupSyncDir = nodeS3Path + "/" + syncDir + "/" + keyspace + "/" + latestAvailableSnapshotFolder + "/" + columnFamily for files in getNewlyAddedFiles(incrementalColumnFamilyPath,nodeS3Path,syncDir,keyspace,latestAvailableSnapshotFolder,columnFamily,action): os.chdir(incrementalColumnFamilyPath) s3SyncCommand = "aws s3 cp " + files + " " + incrementalBackupSyncDir + "/" + files print "Syncing Differential Incremental Backup: <Local-2-S3>" print (files) print "Executing: " + s3SyncCommand + " to sync incremental backup of keyspace " + keyspace + " for cloumn family " + columnFamily os.system(s3SyncCommand)
def syncIncrementalBackupIntoLastSnapshotFolder(): validateKeyspace() createIncrementalBackup() nodeS3Path = getNodeBackupS3Path() latestAvailableSnapshotFolder = getLatestAvailableSnapshotFolderNameAtS3() syncDir = 'snapshots' action = 'incremental' # It's important for deciding sync or snapshots directory for uploading files for incrementalColumnFamilyPath in getIncrementalBackupDirectoriesPath(): keyspacePath = incrementalColumnFamilyPath.split("/backups")[0] keyspaceAndColumnFamilies = keyspacePath.split(cassandraDataDir())[1] columnFamily = keyspaceAndColumnFamilies.split("/")[2] incrementalBackupSyncDir = nodeS3Path + "/" + syncDir + "/" + keyspace + "/" + latestAvailableSnapshotFolder + "/" + columnFamily for files in getNewlyAddedFiles(incrementalColumnFamilyPath, nodeS3Path, syncDir, keyspace, latestAvailableSnapshotFolder, columnFamily, action): os.chdir(incrementalColumnFamilyPath) s3SyncCommand = "aws s3 cp " + files + " " + incrementalBackupSyncDir + "/" + files print "Syncing Differential Incremental Backup: <Local-2-S3>" print(files) print "Executing: " + s3SyncCommand + " to sync incremental backup of keyspace " + keyspace + " for cloumn family " + columnFamily os.system(s3SyncCommand)