def list_bucket(self,
name,
marker=None,
prefix=None,
delimiter=None,
**config):
"Return an iterator over all the keys in bucket named ``name``"
bucket = self._get_bucket(name)
marker = marker or config.get('list_marker') or ''
prefix = prefix or config.get('list_prefix') or ''
delimiter = delimiter or config.get('list_delimiter') or ''
more_results = True
k = None
while more_results:
log.info("Listing keys at %s/%s from marker: %s", name, prefix,
marker)
try:
r = bucket.get_all_keys(marker=marker,
prefix=prefix,
delimiter=delimiter)
for k in r:
yield k.name
if k:
marker = k.name
more_results = r.is_truncated
except BotoServerError, e:
raise FunnelError("Failed to list bucket: %s" % name, key=name)
except (IncompleteRead, SocketError, BotoClientError), e:
log.warning("Caught exception: %r.\nRetrying..." % e)
def drop_bucket(self, name, **config):
"Delete bucket named ``name``"
conn = self._get_conn()
try:
conn.delete_bucket(name)
log.info("Deleted bucket: %s" % name)
except BotoServerError, e:
raise FunnelError("Bucket could not be deleted: %s" % name, key=name)
def create_bucket(self, name, **config):
"Create bucket named ``name``"
conn = self._get_conn()
try:
b = conn.create_bucket(name)
log.info("Created bucket: %s" % name)
except BotoServerError, e:
raise FunnelError("Bucket could not be created: %s" % name, key=name)
def get_bucket(self, name):
"Get an S3 bucket instance with the given ``name`` (cached)"
bucket = self.buckets.get(name)
if bucket: return bucket
conn = self.get_conn()
log.debug("Getting bucket instance: %s" % name)
try:
bucket = conn.get_bucket(name)
except BotoServerError, e:
raise FunnelError("Bucket not found: %s" % name, key=name)