def downloadTag(srcCluster, dstCluster, tagName, dstDir=None, baseDir=None): """ srcCluster - Cluster to download the tag from dstCluster - Cluster to download the tag to (this needs to be 'local' for now) tagName - The name of the tag to download dstDir - The destination directory to put downloaded data to. If None, dstDir is assumed to be dstCluster['config']['dirs.upload_dir'] baseDir - When we download a tag we replicate its directory structure, baseDir allows us to remove some portion of the prefix dir in downloading. If baseDir is None then srcCluster['config']['dirs.upload_dir'] is assumed to be the baseDir Neither dstDir or baseDir should consider the 'tagname' as part of their name. This may change in the future though if we want to allow downloading to a new tag name TODO: Consider compressing tags on the remote side before a transfer. It is possible it should be part of the download process. Alternatively it might make more sense for the compression to be part of another process. """ if dstDir is None: dstDir = os.path.join(dstCluster['config']['dirs.upload_dir'], tagName) if baseDir is None: baseDir = srcCluster['config']['dirs.upload_dir'] # # Get the list of files tagData = queryTag('localhost', srcCluster['cluster_name'], [tagName]) # # Some tags have a tag_base_dir metadata element which shows how much of the path # to cut off when transfering. However, it is not guaranteed that every file # in the tag will be in thise base directory, it is meant more as a guide. # If this metadata element exists we want to group the download into 2 steps. # 1) Download all files that exist in the tag_base_dir and cut off tag_base_dir # in the download # 2) The rest baseDirFiles, nonBaseDirFiles = partitionFiles(tagData('files'), baseDir) if baseDirFiles: # # Write those files out to a temporary file so we can use it with --files-from in rysnc tmpFName = os.path.join(dstCluster['config']['general.secure_tmp'], 'rsync-tmp-' + str(time.time())) fout = open(tmpFName, 'w') fout.writelines([f + '\n' for f in baseDirFiles]) fout.close() # # Do the rsync try: makeDirsOnCluster(dstCluster, [dstDir]) except TryError, err: errorPrint('Caught TryError, ignoring for now: %s - %s ' % (err.msg, str(err.result))) rsyncOptions = ' '.join([dstCluster['config']['rsync.options'], '--files-from=' + tmpFName]) rsyncFromEx(srcCluster['master']['public_dns'], baseDir, dstDir, rsyncOptions=rsyncOptions, user=srcCluster['config']['rsync.user'], log=True) os.unlink(tmpFName)
def downloadTag(srcCluster, dstCluster, tagName, dstDir=None, baseDir=None): """ srcCluster - Cluster to download the tag from dstCluster - Cluster to download the tag to (this needs to be 'local' for now) tagName - The name of the tag to download dstDir - The destination directory to put downloaded data to. If None, dstDir is assumed to be dstCluster['config']['dirs.upload_dir'] baseDir - When we download a tag we replicate its directory structure, baseDir allows us to remove some portion of the prefix dir in downloading. If baseDir is None then srcCluster['config']['dirs.upload_dir'] is assumed to be the baseDir Neither dstDir or baseDir should consider the 'tagname' as part of their name. This may change in the future though if we want to allow downloading to a new tag name TODO: Consider compressing tags on the remote side before a transfer. It is possible it should be part of the download process. Alternatively it might make more sense for the compression to be part of another process. """ if dstDir is None: dstDir = os.path.join(dstCluster['config']['dirs.upload_dir'], tagName) if baseDir is None: baseDir = srcCluster['config']['dirs.upload_dir'] # # Get the list of files tagData = queryTag('localhost', srcCluster['cluster_name'], [tagName]) # # Some tags have a tag_base_dir metadata element which shows how much of the path # to cut off when transfering. However, it is not guaranteed that every file # in the tag will be in thise base directory, it is meant more as a guide. # If this metadata element exists we want to group the download into 2 steps. # 1) Download all files that exist in the tag_base_dir and cut off tag_base_dir # in the download # 2) The rest baseDirFiles, nonBaseDirFiles = partitionFiles(tagData('files'), baseDir) if baseDirFiles: # # Write those files out to a temporary file so we can use it with --files-from in rysnc tmpFName = os.path.join(dstCluster['config']['general.secure_tmp'], 'rsync-tmp-' + str(time.time())) fout = open(tmpFName, 'w') fout.writelines([f + '\n' for f in baseDirFiles]) fout.close() # # Do the rsync try: makeDirsOnCluster(dstCluster, [dstDir]) except TryError, err: errorPrint('Caught TryError, ignoring for now: %s - %s ' % (err.msg, str(err.result))) rsyncOptions = ' '.join([ dstCluster['config']['rsync.options'], '--files-from=' + tmpFName ]) rsyncFromEx(srcCluster['master']['public_dns'], baseDir, dstDir, rsyncOptions=rsyncOptions, user=srcCluster['config']['rsync.user'], log=True) os.unlink(tmpFName)
os.unlink(tmpFName) if nonBaseDirFiles: # # Write those files out to a temporary file so we can use it with --files-from in rysnc tmpFName = os.path.join(dstCluster['config']['general.secure_tmp'], 'rsync-tmp-' + str(time.time())) fout = open(tmpFName, 'w') fout.writelines([f + '\n' for f in nonBaseDirFiles]) fout.close() # # Do the rsync try: makeDirsOnCluster(dstCluster, [dstDir]) except TryError, err: errorPrint('Caught TryError, ignoring for now: %s - %s ' % (err.msg, str(err.result))) rsyncOptions = ' '.join([srcCluster['config']['rsync.options'], '--files-from=' + tmpFName]) rsyncFromEx(srcCluster['master']['public_dns'], '/', dstDir, rsyncOptions=rsyncOptions, user=srcCluster['config']['rsync.user'], log=True) os.unlink(tmpFName) return config.configFromMap({'files': [os.path.join(dstDir, makePathRelative(f)) for f in baseDirFiles + nonBaseDirFiles], 'metadata.tag_base_dir': dstDir})
fout.writelines([f + '\n' for f in nonBaseDirFiles]) fout.close() # # Do the rsync try: makeDirsOnCluster(dstCluster, [dstDir]) except TryError, err: errorPrint('Caught TryError, ignoring for now: %s - %s ' % (err.msg, str(err.result))) rsyncOptions = ' '.join([ srcCluster['config']['rsync.options'], '--files-from=' + tmpFName ]) rsyncFromEx(srcCluster['master']['public_dns'], '/', dstDir, rsyncOptions=rsyncOptions, user=srcCluster['config']['rsync.user'], log=True) os.unlink(tmpFName) return config.configFromMap({ 'files': [ os.path.join(dstDir, makePathRelative(f)) for f in baseDirFiles + nonBaseDirFiles ], 'metadata.tag_base_dir': dstDir })