def _autocontainer_loop(self, account, marker=None, limit=None,
concurrency=1, **kwargs):
from functools import partial
container_marker = self.flatns_manager(marker) if marker else None
count = 0
kwargs['pool_manager'] = get_pool_manager(
pool_maxsize=concurrency * 2)
# Start to list contents at 'marker' inside the last visited container
if container_marker:
for element in depaginate(
self.app.client_manager.storage.object_list,
listing_key=lambda x: x['objects'],
marker_key=lambda x: x.get('next_marker'),
truncated_key=lambda x: x['truncated'],
account=account, container=container_marker,
marker=marker, **kwargs):
count += 1
yield element
if limit and count >= limit:
return
pool = GreenPool(concurrency)
for object_list in pool.imap(
partial(self._list_autocontainer_objects,
account=account, **kwargs),
depaginate(self.app.client_manager.storage.container_list,
item_key=lambda x: x[0],
account=account,
marker=container_marker)):
for element in object_list:
count += 1
yield element
if limit and count >= limit:
return
def take_action(self, parsed_args):
self.log.debug('take_action(%s)', parsed_args)
kwargs = {'reqid': request_id(prefix='CLI-BUCKET-')}
if parsed_args.prefix:
kwargs['prefix'] = parsed_args.prefix
if parsed_args.marker:
kwargs['marker'] = parsed_args.marker
if parsed_args.limit:
kwargs['limit'] = parsed_args.limit
account = self.app.client_manager.account
acct_client = self.app.client_manager.storage.account
if parsed_args.full_listing:
listing = depaginate(acct_client.bucket_list,
listing_key=lambda x: x['listing'],
marker_key=lambda x: x.get('next_marker'),
truncated_key=lambda x: x['truncated'],
account=account,
**kwargs)
else:
acct_meta = acct_client.bucket_list(account, **kwargs)
listing = acct_meta['listing']
columns = ('Name', 'Bytes', 'Objects', 'Mtime')
return columns, ((v['name'], v['bytes'], v['objects'], v['mtime'])
for v in listing)
def execute(self, use_precessed_versions=True, **kwargs):
"""
Match then apply the set of rules of the lifecycle configuration
on all objects of the container (and its relatives, if recursive
mode is enabled).
:returns: tuples of (object metadata, rule name, action, status)
:rtype: generator of 4-tuples
:notice: the results must be consumed or the rules won't be applied.
"""
if use_precessed_versions:
self.processed_versions = ProcessedVersions()
results = self.process_container(self.container, **kwargs)
for res in results:
yield res
if self.recursive:
for container in depaginate(self.api.container_list,
item_key=lambda x: x[0],
marker_key=lambda x: x[-1][0],
account=self.account,
prefix=self.container +
CH_ENCODED_SEPARATOR):
results = self.process_container(container, **kwargs)
for res in results:
yield res
self.processed_versions = None
def execute(self, **kwargs):
"""
Match then apply the set of rules of the lifecycle configuration
on all objects of the container.
:returns: tuples of (object metadata, rule name, action, status)
:rtype: generator of 4-tuples
:notice: you must consume the results or the rules won't be applied.
"""
for obj_meta in depaginate(self.api.object_list,
listing_key=lambda x: x['objects'],
marker_key=lambda x: x.get('next_marker'),
truncated_key=lambda x: x['truncated'],
account=self.account,
container=self.container,
properties=True,
versions=True,
**kwargs):
try:
results = self.apply(obj_meta, **kwargs)
for res in results:
yield res
except Exception as exc:
self.logger.warn(
"Failed to apply lifecycle rules on %s/%s/%s: %s",
self.account, self.container, obj_meta['name'], exc)
yield obj_meta, "n/a", "n/a", exc
def list_objects(container):
objects_iter = depaginate(API.object_list,
listing_key=lambda x: x['objects'],
marker_key=lambda x: x.get('next_marker'),
truncated_key=lambda x: x['truncated'],
account=ACCOUNT,
container=container,
limit=5000)
return [obj['name'] for obj in objects_iter]
def _list_autocontainer_objects(self, container, account, **kwargs):
if not self.flatns_manager.verify(container):
self.log.debug("Container %s is not an autocontainer",
container)
return list()
self.log.debug("Listing autocontainer %s", container)
object_list = []
for i in depaginate(
self.app.client_manager.storage.object_list,
listing_key=lambda x: x['objects'],
marker_key=lambda x: x['objects'][-1]['name'],
account=account, container=container, **kwargs):
object_list.append(i)
return object_list
def meta2_index_fetch_all(self, volume_id, **kwargs):
"""
A wrapper around meta2_index_fetch that loops until no more records
are available, returning all the records in a certain volume's index.
WARNING: For testing purposes only
"""
return depaginate(
self.meta2_index_fetch,
volume_id=volume_id,
listing_key=lambda x: x['records'],
truncated_key=lambda x: x['truncated'],
# The following is only called when the list is truncated
# So we can assume there are records in the list
marker_key=lambda x: x['records'][-1]['container_url'],
**kwargs)
def process_container(self, container, **kwargs):
"""
Match then apply the set of rules of the lifecycle configuration
on all objects of the container.
:returns: tuples of (object metadata, rule name, action, status)
:rtype: generator of 4-tuples
:notice: the results must be consumed or the rules won't be applied.
"""
for obj_meta in depaginate(self.api.object_list,
listing_key=lambda x: x['objects'],
marker_key=lambda x: x.get('next_marker'),
truncated_key=lambda x: x['truncated'],
account=self.account,
container=container,
properties=True,
versions=True,
**kwargs):
try:
# Save the name of the object as it is in the container,
# for later use.
obj_meta['orig_name'] = obj_meta['name']
# And reconstruct the name of the object as it is
# when shown to the final user.
if self.recursive and CH_ENCODED_SEPARATOR in container:
obj_meta['name'] = container.split(CH_ENCODED_SEPARATOR,
1)[1]
obj_meta['name'].replace(CH_ENCODED_SEPARATOR,
CH_SEPARATOR)
obj_meta['name'] += CH_SEPARATOR + obj_meta['orig_name']
obj_meta['container'] = container
if self.processed_versions is not None \
and self.processed_versions.is_already_processed(
obj_meta, **kwargs):
continue
results = self.apply(obj_meta, **kwargs)
for res in results:
yield res
except Exception as exc:
self.logger.warn(
"Failed to apply lifecycle rules on %s/%s/%s: %s",
self.account, self.container, obj_meta['name'], exc)
yield obj_meta, "n/a", "n/a", exc
if self.processed_versions is not None:
self.processed_versions.save_object(obj_meta, **kwargs)
def _list_autocontainer_objects(self, container, account, **kwargs):
object_list = []
if not self.flatns_manager.verify(container):
self.log.debug("Container %s is not an autocontainer", container)
return object_list
self.log.debug("Listing autocontainer %s", container)
try:
for i in depaginate(self.app.client_manager.storage.object_list,
listing_key=lambda x: x['objects'],
marker_key=lambda x: x.get('next_marker'),
truncated_key=lambda x: x['truncated'],
account=account,
container=container,
**kwargs):
object_list.append(i)
except exceptions.OioException as err:
self.log.warn('Listing may be incomplete: container %s: %s',
container, err)
return object_list
def take_action(self, parsed_args):
self.log.debug('take_action(%s)', parsed_args)
super(ListObject, self).take_action(parsed_args)
kwargs = {}
if parsed_args.prefix:
kwargs['prefix'] = parsed_args.prefix
if parsed_args.marker:
kwargs['marker'] = parsed_args.marker
if parsed_args.end_marker:
kwargs['end_marker'] = parsed_args.end_marker
if parsed_args.delimiter:
kwargs['delimiter'] = parsed_args.delimiter
if parsed_args.limit and not parsed_args.full_listing:
kwargs['limit'] = parsed_args.limit
if parsed_args.long_listing:
kwargs['properties'] = True
if parsed_args.versions:
kwargs['versions'] = True
if parsed_args.local:
kwargs['local'] = True
if parsed_args.concurrency:
kwargs['concurrency'] = parsed_args.concurrency
if parsed_args.attempts:
kwargs['request_attempts'] = parsed_args.attempts
account = self.app.client_manager.account
if parsed_args.auto:
obj_gen = self._autocontainer_loop(account, **kwargs)
else:
container = parsed_args.container
if parsed_args.full_listing:
obj_gen = depaginate(
self.app.client_manager.storage.object_list,
listing_key=lambda x: x['objects'],
marker_key=lambda x: x.get('next_marker'),
truncated_key=lambda x: x['truncated'],
account=account, container=container, **kwargs)
else:
resp = self.app.client_manager.storage.object_list(
account, container, **kwargs)
obj_gen = resp['objects']
if parsed_args.long_listing:
from oio.common.timestamp import Timestamp
def _format_props(props):
prop_list = ["%s=%s" % (k, v) for k, v
in props.items()]
if parsed_args.formatter == 'table':
prop_string = "\n".join(prop_list)
elif parsed_args.formatter in ('value', 'csv'):
prop_string = " ".join(prop_list)
else:
prop_string = props
return prop_string
def _gen_results(objects):
for obj in objects:
result = (obj['name'], obj['size'],
obj['hash'], obj['version'],
obj['deleted'], obj['mime_type'],
Timestamp(obj['ctime']).isoformat,
obj['policy'],
_format_props(obj.get('properties', {})))
yield result
results = _gen_results(obj_gen)
columns = ('Name', 'Size', 'Hash', 'Version', 'Deleted',
'Content-Type', 'Last-Modified', 'Policy', 'Properties')
else:
results = ((obj['name'],
obj['size'] if not obj['deleted'] else 'deleted',
obj['hash'],
obj['version'])
for obj in obj_gen)
columns = ('Name', 'Size', 'Hash', 'Version')
return (columns, results)
def take_action(self, parsed_args):
self.log.debug('take_action(%s)', parsed_args)
kwargs = {'reqid': request_id(prefix='CLI-BUCKET-')}
if parsed_args.prefix:
kwargs['prefix'] = parsed_args.prefix
if parsed_args.marker:
kwargs['marker'] = parsed_args.marker
if parsed_args.limit:
kwargs['limit'] = parsed_args.limit
account = self.app.client_manager.account
acct_client = self.app.client_manager.storage.account
storage = self.app.client_manager.storage
if parsed_args.full_listing:
listing = depaginate(acct_client.bucket_list,
listing_key=lambda x: x['listing'],
marker_key=lambda x: x.get('next_marker'),
truncated_key=lambda x: x['truncated'],
account=account,
**kwargs)
else:
acct_meta = acct_client.bucket_list(account, **kwargs)
listing = acct_meta['listing']
columns = (
'Name',
'Bytes',
'Objects',
'Mtime',
)
def versioning(bucket):
try:
data = storage.container_get_properties(account,
bucket,
reqid=kwargs['reqid'])
except NoSuchContainer:
self.log.info('Bucket %s does not exist', bucket)
return "Error"
sys = data['system']
# WARN it doe not reflect namespace versioning if enabled
status = sys.get(M2_PROP_VERSIONING_POLICY, None)
if status is None or int(status) == 0:
return "Suspended"
else:
return "Enabled"
if parsed_args.versioning:
columns += ('Versioning', )
def enrich(listing):
for v in listing:
v['versioning'] = versioning(v['name'])
yield v
listing = enrich(listing)
return columns, ([v[k.lower()] for k in columns] for v in listing)
