def s3_syncdownload_dir(bucketname, s3prefix, local_dir, loglevel='INFO', delete=False,
pretend=False):
"""
Sync a S3 prefix from a S3 bucket into a local directory. Uses the same
method as the :func:`s3_is_same_file` task to determine if a local file
differs from a file on S3.
:param bucketname: Name of an S3 bucket.
:param s3prefix: The S3 prefix to use for the uploaded files.
:param local_dir: The local directory to sync to S3.
:param loglevel:
Controls the amount of output:
QUIET --- No output.
INFO --- Only produce output for changes.
DEBUG --- One line of output for each file.
Defaults to "INFO".
:param delete:
Delete local files that are not present in ``s3prefix``.
:param pretend:
Do not change anything. With ``verbosity=2``, this gives a good
overview of the changes applied by running the task.
"""
log = configureStreamLoggerForTask(__name__, 's3_syncupload_dir',
getLoglevelFromString(loglevel))
local_dir = abspath(expanduser(local_dir))
delete = parse_bool(delete)
pretend = parse_bool(pretend)
bucket = S3ConnectionWrapper.get_bucket_using_pattern(bucketname)
if pretend:
log.info('Running in pretend mode. No changes are made.')
for syncfile in S3Sync(bucket, local_dir, s3prefix).iterfiles():
logname = 'LocalFS:{0}'.format(syncfile.localpath)
if syncfile.both_exists():
if syncfile.etag_matches_localfile():
log.debug('UNCHANGED %s', logname)
else:
if not pretend:
log.debug('Downloading %s', logname)
syncfile.download_s3file_to_localfile()
log.info('UPDATED %s', logname)
elif syncfile.s3exists:
if not pretend:
log.debug('Downloading %s', logname)
syncfile.download_s3file_to_localfile()
log.info('CREATED %s', logname)
else:
if delete:
if not pretend:
remove(syncfile.localpath)
log.info('DELETED %s', logname)
else:
log.debug('NOT DELETED %s (it does not exist on S3)', syncfile.localpath)
def s3_delete(bucketname, keyname, noconfirm=False):
"""
Remove a "file" from the given bucket.
:param bucketname: Name of an S3 bucket.
:param keyname: The key to remove (In filesystem terms: absolute file path).
:param noconfirm:
If this is ``True``, we will not ask for confirmation before
removing the key. Defaults to ``False``.
"""
bucket = S3ConnectionWrapper.get_bucket_using_pattern(bucketname)
s3file = S3File.raw(bucket, keyname)
if not parse_bool(noconfirm):
if not confirm('Remove {0}?'.format(keyname)):
abort('Aborted')
s3file.delete()
def ec2_rsync_download_dir(remote_dir, local_dir, rsync_args="-av", noconfirm=False):
"""
Sync the contents of ``remote_dir`` into ``local_dir``. E.g.: if ``remote_dir`` is
``/etc``, and ``local_dir`` is ``/tmp``, the ``/tmp/etc`` will be created on the local
host, and filled with all files in ``/etc`` on the EC2 instance.
:param remote_dir: The remote directory to download into local_dir.
:param local_dir: The local directory.
:param rsync_args: Arguments for ``rsync``. Defaults to ``-av``.
:param noconfirm:
If this is ``True``, we will not ask for confirmation before
proceeding with the operation. Defaults to ``False``.
"""
kwargs = dict(remote_dir=remote_dir, local_dir=local_dir, rsync_args=rsync_args)
if not parse_bool(noconfirm):
instancewrapper = Ec2InstanceWrapper.get_from_host_string()
print "Are you sure you want to run:"
print " ", ec2_rsync_download_command(instancewrapper, **kwargs)
if not confirm("Proceed?"):
abort("Aborted")
ec2_rsync_download(**kwargs)
