def syncSnapshotContentToS3SyncDirAndCreateSnapshot(nodeS3Path,keyspace,snapshotDate,s3SnapshotFolderName):
# Get Current working directory
scriptExecutionPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory
snapshotDirColumnFamilyPaths = getLocalColumnFamilyPathsToSync(keyspace,snapshotDate)
syncDir = 'sync_dir'
action = 'backup' # It's important for deciding sync or snapshots directory for uploading files
s3SnapshotFolderName = datetime.datetime.strftime(datetime.datetime.now(), '%Y%m%d')
for snapshotDirColumnFamilyPath in snapshotDirColumnFamilyPaths:
columnFamily=getColumnFamilyName(snapshotDirColumnFamilyPath)
s3SyncDir = nodeS3Path + "/sync_dir/"+keyspace + "/"+columnFamily
os.chdir(snapshotDirColumnFamilyPath)
for newFile in getNewlyAddedFiles(snapshotDirColumnFamilyPath,nodeS3Path,syncDir,keyspace,s3SnapshotFolderName,columnFamily,action):
copyFileToS3(newFile,s3SyncDir)
for toBeRemovedFile in getListOfDeletedFiles(snapshotDirColumnFamilyPath,nodeS3Path,syncDir,keyspace,columnFamily,action,s3SnapshotFolderName):
deleteFilesFromS3(toBeRemovedFile,s3SyncDir)
os.chdir(scriptExecutionPath)
createSnapshotAtS3(nodeS3Path, keyspace, snapshotDate,columnFamily,s3SyncDir)
def syncIncrementalBackupIntoLastSnapshotFolder():
validateKeyspace()
createIncrementalBackup()
nodeS3Path = getNodeBackupS3Path()
latestAvailableSnapshotFolder = getLatestAvailableSnapshotFolderNameAtS3()
syncDir = 'snapshots'
action = 'incremental' # It's important for deciding sync or snapshots directory for uploading files
for incrementalColumnFamilyPath in getIncrementalBackupDirectoriesPath():
keyspacePath=incrementalColumnFamilyPath.split("/backups")[0]
keyspaceAndColumnFamilies=keyspacePath.split(cassandraDataDir())[1]
columnFamily=keyspaceAndColumnFamilies.split("/")[2]
incrementalBackupSyncDir = nodeS3Path + "/" + syncDir + "/" + keyspace + "/" + latestAvailableSnapshotFolder + "/" + columnFamily
for files in getNewlyAddedFiles(incrementalColumnFamilyPath,nodeS3Path,syncDir,keyspace,latestAvailableSnapshotFolder,columnFamily,action):
os.chdir(incrementalColumnFamilyPath)
s3SyncCommand = "aws s3 cp " + files + " " + incrementalBackupSyncDir + "/" + files
print "Syncing Differential Incremental Backup: <Local-2-S3>"
print (files)
print "Executing: " + s3SyncCommand + " to sync incremental backup of keyspace " + keyspace + " for cloumn family " + columnFamily
os.system(s3SyncCommand)
def syncIncrementalBackupIntoLastSnapshotFolder():
validateKeyspace()
createIncrementalBackup()
nodeS3Path = getNodeBackupS3Path()
latestAvailableSnapshotFolder = getLatestAvailableSnapshotFolderNameAtS3()
syncDir = 'snapshots'
action = 'incremental' # It's important for deciding sync or snapshots directory for uploading files
for incrementalColumnFamilyPath in getIncrementalBackupDirectoriesPath():
keyspacePath = incrementalColumnFamilyPath.split("/backups")[0]
keyspaceAndColumnFamilies = keyspacePath.split(cassandraDataDir())[1]
columnFamily = keyspaceAndColumnFamilies.split("/")[2]
incrementalBackupSyncDir = nodeS3Path + "/" + syncDir + "/" + keyspace + "/" + latestAvailableSnapshotFolder + "/" + columnFamily
for files in getNewlyAddedFiles(incrementalColumnFamilyPath,
nodeS3Path, syncDir, keyspace,
latestAvailableSnapshotFolder,
columnFamily, action):
os.chdir(incrementalColumnFamilyPath)
s3SyncCommand = "aws s3 cp " + files + " " + incrementalBackupSyncDir + "/" + files
print "Syncing Differential Incremental Backup: <Local-2-S3>"
print(files)
print "Executing: " + s3SyncCommand + " to sync incremental backup of keyspace " + keyspace + " for cloumn family " + columnFamily
os.system(s3SyncCommand)
