def downloadTag(srcCluster, dstCluster, tagName, dstDir=None, baseDir=None):
"""
srcCluster - Cluster to download the tag from
dstCluster - Cluster to download the tag to (this needs to be 'local' for now)
tagName - The name of the tag to download
dstDir - The destination directory to put downloaded data to. If None, dstDir
is assumed to be dstCluster['config']['dirs.upload_dir']
baseDir - When we download a tag we replicate its directory structure, baseDir allows
us to remove some portion of the prefix dir in downloading. If baseDir is None
then srcCluster['config']['dirs.upload_dir'] is assumed to be the baseDir
Neither dstDir or baseDir should consider the 'tagname' as part of their name.
This may change in the future though if we want to allow downloading to a new tag name
TODO: Consider compressing tags on the remote side before a transfer. It is possible it
should be part of the download process. Alternatively it might make more sense for the compression
to be part of another process.
"""
if dstDir is None:
dstDir = os.path.join(dstCluster['config']['dirs.upload_dir'], tagName)
if baseDir is None:
baseDir = srcCluster['config']['dirs.upload_dir']
#
# Get the list of files
tagData = queryTag('localhost', srcCluster['cluster_name'], [tagName])
#
# Some tags have a tag_base_dir metadata element which shows how much of the path
# to cut off when transfering. However, it is not guaranteed that every file
# in the tag will be in thise base directory, it is meant more as a guide.
# If this metadata element exists we want to group the download into 2 steps.
# 1) Download all files that exist in the tag_base_dir and cut off tag_base_dir
# in the download
# 2) The rest
baseDirFiles, nonBaseDirFiles = partitionFiles(tagData('files'), baseDir)
if baseDirFiles:
#
# Write those files out to a temporary file so we can use it with --files-from in rysnc
tmpFName = os.path.join(dstCluster['config']['general.secure_tmp'], 'rsync-tmp-' + str(time.time()))
fout = open(tmpFName, 'w')
fout.writelines([f + '\n' for f in baseDirFiles])
fout.close()
#
# Do the rsync
try:
makeDirsOnCluster(dstCluster, [dstDir])
except TryError, err:
errorPrint('Caught TryError, ignoring for now: %s - %s ' % (err.msg, str(err.result)))
rsyncOptions = ' '.join([dstCluster['config']['rsync.options'], '--files-from=' + tmpFName])
rsyncFromEx(srcCluster['master']['public_dns'],
baseDir,
dstDir,
rsyncOptions=rsyncOptions,
user=srcCluster['config']['rsync.user'],
log=True)
os.unlink(tmpFName)
def downloadTag(srcCluster, dstCluster, tagName, dstDir=None, baseDir=None):
"""
srcCluster - Cluster to download the tag from
dstCluster - Cluster to download the tag to (this needs to be 'local' for now)
tagName - The name of the tag to download
dstDir - The destination directory to put downloaded data to. If None, dstDir
is assumed to be dstCluster['config']['dirs.upload_dir']
baseDir - When we download a tag we replicate its directory structure, baseDir allows
us to remove some portion of the prefix dir in downloading. If baseDir is None
then srcCluster['config']['dirs.upload_dir'] is assumed to be the baseDir
Neither dstDir or baseDir should consider the 'tagname' as part of their name.
This may change in the future though if we want to allow downloading to a new tag name
TODO: Consider compressing tags on the remote side before a transfer. It is possible it
should be part of the download process. Alternatively it might make more sense for the compression
to be part of another process.
"""
if dstDir is None:
dstDir = os.path.join(dstCluster['config']['dirs.upload_dir'], tagName)
if baseDir is None:
baseDir = srcCluster['config']['dirs.upload_dir']
#
# Get the list of files
tagData = queryTag('localhost', srcCluster['cluster_name'], [tagName])
#
# Some tags have a tag_base_dir metadata element which shows how much of the path
# to cut off when transfering. However, it is not guaranteed that every file
# in the tag will be in thise base directory, it is meant more as a guide.
# If this metadata element exists we want to group the download into 2 steps.
# 1) Download all files that exist in the tag_base_dir and cut off tag_base_dir
# in the download
# 2) The rest
baseDirFiles, nonBaseDirFiles = partitionFiles(tagData('files'), baseDir)
if baseDirFiles:
#
# Write those files out to a temporary file so we can use it with --files-from in rysnc
tmpFName = os.path.join(dstCluster['config']['general.secure_tmp'],
'rsync-tmp-' + str(time.time()))
fout = open(tmpFName, 'w')
fout.writelines([f + '\n' for f in baseDirFiles])
fout.close()
#
# Do the rsync
try:
makeDirsOnCluster(dstCluster, [dstDir])
except TryError, err:
errorPrint('Caught TryError, ignoring for now: %s - %s ' %
(err.msg, str(err.result)))
rsyncOptions = ' '.join([
dstCluster['config']['rsync.options'], '--files-from=' + tmpFName
])
rsyncFromEx(srcCluster['master']['public_dns'],
baseDir,
dstDir,
rsyncOptions=rsyncOptions,
user=srcCluster['config']['rsync.user'],
log=True)
os.unlink(tmpFName)
os.unlink(tmpFName)
if nonBaseDirFiles:
#
# Write those files out to a temporary file so we can use it with --files-from in rysnc
tmpFName = os.path.join(dstCluster['config']['general.secure_tmp'], 'rsync-tmp-' + str(time.time()))
fout = open(tmpFName, 'w')
fout.writelines([f + '\n' for f in nonBaseDirFiles])
fout.close()
#
# Do the rsync
try:
makeDirsOnCluster(dstCluster, [dstDir])
except TryError, err:
errorPrint('Caught TryError, ignoring for now: %s - %s ' % (err.msg, str(err.result)))
rsyncOptions = ' '.join([srcCluster['config']['rsync.options'], '--files-from=' + tmpFName])
rsyncFromEx(srcCluster['master']['public_dns'],
'/',
dstDir,
rsyncOptions=rsyncOptions,
user=srcCluster['config']['rsync.user'],
log=True)
os.unlink(tmpFName)
return config.configFromMap({'files': [os.path.join(dstDir, makePathRelative(f)) for f in baseDirFiles + nonBaseDirFiles],
'metadata.tag_base_dir': dstDir})
fout.writelines([f + '\n' for f in nonBaseDirFiles])
fout.close()
#
# Do the rsync
try:
makeDirsOnCluster(dstCluster, [dstDir])
except TryError, err:
errorPrint('Caught TryError, ignoring for now: %s - %s ' %
(err.msg, str(err.result)))
rsyncOptions = ' '.join([
srcCluster['config']['rsync.options'], '--files-from=' + tmpFName
])
rsyncFromEx(srcCluster['master']['public_dns'],
'/',
dstDir,
rsyncOptions=rsyncOptions,
user=srcCluster['config']['rsync.user'],
log=True)
os.unlink(tmpFName)
return config.configFromMap({
'files': [
os.path.join(dstDir, makePathRelative(f))
for f in baseDirFiles + nonBaseDirFiles
],
'metadata.tag_base_dir':
dstDir
})
