def s3_syncdownload_dir(bucketname, s3prefix, local_dir, loglevel='INFO', delete=False, pretend=False): """ Sync a S3 prefix from a S3 bucket into a local directory. Uses the same method as the :func:`s3_is_same_file` task to determine if a local file differs from a file on S3. :param bucketname: Name of an S3 bucket. :param s3prefix: The S3 prefix to use for the uploaded files. :param local_dir: The local directory to sync to S3. :param loglevel: Controls the amount of output: QUIET --- No output. INFO --- Only produce output for changes. DEBUG --- One line of output for each file. Defaults to "INFO". :param delete: Delete local files that are not present in ``s3prefix``. :param pretend: Do not change anything. With ``verbosity=2``, this gives a good overview of the changes applied by running the task. """ log = configureStreamLoggerForTask(__name__, 's3_syncupload_dir', getLoglevelFromString(loglevel)) local_dir = abspath(expanduser(local_dir)) delete = parse_bool(delete) pretend = parse_bool(pretend) bucket = S3ConnectionWrapper.get_bucket_using_pattern(bucketname) if pretend: log.info('Running in pretend mode. No changes are made.') for syncfile in S3Sync(bucket, local_dir, s3prefix).iterfiles(): logname = 'LocalFS:{0}'.format(syncfile.localpath) if syncfile.both_exists(): if syncfile.etag_matches_localfile(): log.debug('UNCHANGED %s', logname) else: if not pretend: log.debug('Downloading %s', logname) syncfile.download_s3file_to_localfile() log.info('UPDATED %s', logname) elif syncfile.s3exists: if not pretend: log.debug('Downloading %s', logname) syncfile.download_s3file_to_localfile() log.info('CREATED %s', logname) else: if delete: if not pretend: remove(syncfile.localpath) log.info('DELETED %s', logname) else: log.debug('NOT DELETED %s (it does not exist on S3)', syncfile.localpath)
def s3_delete(bucketname, keyname, noconfirm=False): """ Remove a "file" from the given bucket. :param bucketname: Name of an S3 bucket. :param keyname: The key to remove (In filesystem terms: absolute file path). :param noconfirm: If this is ``True``, we will not ask for confirmation before removing the key. Defaults to ``False``. """ bucket = S3ConnectionWrapper.get_bucket_using_pattern(bucketname) s3file = S3File.raw(bucket, keyname) if not parse_bool(noconfirm): if not confirm('Remove {0}?'.format(keyname)): abort('Aborted') s3file.delete()
def ec2_rsync_download_dir(remote_dir, local_dir, rsync_args="-av", noconfirm=False): """ Sync the contents of ``remote_dir`` into ``local_dir``. E.g.: if ``remote_dir`` is ``/etc``, and ``local_dir`` is ``/tmp``, the ``/tmp/etc`` will be created on the local host, and filled with all files in ``/etc`` on the EC2 instance. :param remote_dir: The remote directory to download into local_dir. :param local_dir: The local directory. :param rsync_args: Arguments for ``rsync``. Defaults to ``-av``. :param noconfirm: If this is ``True``, we will not ask for confirmation before proceeding with the operation. Defaults to ``False``. """ kwargs = dict(remote_dir=remote_dir, local_dir=local_dir, rsync_args=rsync_args) if not parse_bool(noconfirm): instancewrapper = Ec2InstanceWrapper.get_from_host_string() print "Are you sure you want to run:" print " ", ec2_rsync_download_command(instancewrapper, **kwargs) if not confirm("Proceed?"): abort("Aborted") ec2_rsync_download(**kwargs)