def s3_cleanup(glob_path, time_old, dry_run=False, conf_paths=None):
"""Delete all files older than *time_old* in *path*.
If *dry_run* is ``True``, then just log the files that need to be
deleted without actually deleting them
"""
runner = EMRJobRunner(conf_paths=conf_paths)
s3_conn = runner.make_s3_conn()
log.info("Deleting all files in %s that are older than %s" % (glob_path, time_old))
for path in runner.ls(glob_path):
bucket_name, key_name = parse_s3_uri(path)
bucket = _get_bucket(s3_conn, bucket_name)
for key in bucket.list(key_name):
last_modified = iso8601_to_datetime(key.last_modified)
age = datetime.utcnow() - last_modified
if age > time_old:
# Delete it
log.info("Deleting %s; is %s old" % (key.name, age))
if not dry_run:
key.delete()
def s3_cleanup(glob_path, time_old, dry_run=False, conf_paths=None):
"""Delete all files older than *time_old* in *path*.
If *dry_run* is ``True``, then just log the files that need to be
deleted without actually deleting them
"""
runner = EMRJobRunner(conf_paths=conf_paths)
s3_conn = runner.make_s3_conn()
log.info('Deleting all files in %s that are older than %s' %
(glob_path, time_old))
for path in runner.ls(glob_path):
bucket_name, key_name = parse_s3_uri(path)
bucket = _get_bucket(s3_conn, bucket_name)
for key in bucket.list(key_name):
last_modified = iso8601_to_datetime(key.last_modified)
age = datetime.utcnow() - last_modified
if age > time_old:
# Delete it
log.info('Deleting %s; is %s old' % (key.name, age))
if not dry_run:
key.delete()
def assert_bucket_validation(self, boto_version, should_validate):
with patch("boto.Version", boto_version):
s3_conn = Mock()
_get_bucket(s3_conn, "walrus")
s3_conn.get_bucket.assert_called_once_with("walrus", validate=should_validate)
def assert_bucket_validation(self, boto_version, should_validate):
with patch('boto.Version', boto_version):
s3_conn = Mock()
_get_bucket(s3_conn, 'walrus')
s3_conn.get_bucket.assert_called_once_with(
'walrus', validate=should_validate)
