def _autocontainer_loop(self, account, marker=None, limit=None, concurrency=1, **kwargs): from functools import partial container_marker = self.flatns_manager(marker) if marker else None count = 0 kwargs['pool_manager'] = get_pool_manager( pool_maxsize=concurrency * 2) # Start to list contents at 'marker' inside the last visited container if container_marker: for element in depaginate( self.app.client_manager.storage.object_list, listing_key=lambda x: x['objects'], marker_key=lambda x: x.get('next_marker'), truncated_key=lambda x: x['truncated'], account=account, container=container_marker, marker=marker, **kwargs): count += 1 yield element if limit and count >= limit: return pool = GreenPool(concurrency) for object_list in pool.imap( partial(self._list_autocontainer_objects, account=account, **kwargs), depaginate(self.app.client_manager.storage.container_list, item_key=lambda x: x[0], account=account, marker=container_marker)): for element in object_list: count += 1 yield element if limit and count >= limit: return
def take_action(self, parsed_args): self.log.debug('take_action(%s)', parsed_args) kwargs = {'reqid': request_id(prefix='CLI-BUCKET-')} if parsed_args.prefix: kwargs['prefix'] = parsed_args.prefix if parsed_args.marker: kwargs['marker'] = parsed_args.marker if parsed_args.limit: kwargs['limit'] = parsed_args.limit account = self.app.client_manager.account acct_client = self.app.client_manager.storage.account if parsed_args.full_listing: listing = depaginate(acct_client.bucket_list, listing_key=lambda x: x['listing'], marker_key=lambda x: x.get('next_marker'), truncated_key=lambda x: x['truncated'], account=account, **kwargs) else: acct_meta = acct_client.bucket_list(account, **kwargs) listing = acct_meta['listing'] columns = ('Name', 'Bytes', 'Objects', 'Mtime') return columns, ((v['name'], v['bytes'], v['objects'], v['mtime']) for v in listing)
def execute(self, use_precessed_versions=True, **kwargs): """ Match then apply the set of rules of the lifecycle configuration on all objects of the container (and its relatives, if recursive mode is enabled). :returns: tuples of (object metadata, rule name, action, status) :rtype: generator of 4-tuples :notice: the results must be consumed or the rules won't be applied. """ if use_precessed_versions: self.processed_versions = ProcessedVersions() results = self.process_container(self.container, **kwargs) for res in results: yield res if self.recursive: for container in depaginate(self.api.container_list, item_key=lambda x: x[0], marker_key=lambda x: x[-1][0], account=self.account, prefix=self.container + CH_ENCODED_SEPARATOR): results = self.process_container(container, **kwargs) for res in results: yield res self.processed_versions = None
def execute(self, **kwargs): """ Match then apply the set of rules of the lifecycle configuration on all objects of the container. :returns: tuples of (object metadata, rule name, action, status) :rtype: generator of 4-tuples :notice: you must consume the results or the rules won't be applied. """ for obj_meta in depaginate(self.api.object_list, listing_key=lambda x: x['objects'], marker_key=lambda x: x.get('next_marker'), truncated_key=lambda x: x['truncated'], account=self.account, container=self.container, properties=True, versions=True, **kwargs): try: results = self.apply(obj_meta, **kwargs) for res in results: yield res except Exception as exc: self.logger.warn( "Failed to apply lifecycle rules on %s/%s/%s: %s", self.account, self.container, obj_meta['name'], exc) yield obj_meta, "n/a", "n/a", exc
def list_objects(container): objects_iter = depaginate(API.object_list, listing_key=lambda x: x['objects'], marker_key=lambda x: x.get('next_marker'), truncated_key=lambda x: x['truncated'], account=ACCOUNT, container=container, limit=5000) return [obj['name'] for obj in objects_iter]
def _list_autocontainer_objects(self, container, account, **kwargs): if not self.flatns_manager.verify(container): self.log.debug("Container %s is not an autocontainer", container) return list() self.log.debug("Listing autocontainer %s", container) object_list = [] for i in depaginate( self.app.client_manager.storage.object_list, listing_key=lambda x: x['objects'], marker_key=lambda x: x['objects'][-1]['name'], account=account, container=container, **kwargs): object_list.append(i) return object_list
def meta2_index_fetch_all(self, volume_id, **kwargs): """ A wrapper around meta2_index_fetch that loops until no more records are available, returning all the records in a certain volume's index. WARNING: For testing purposes only """ return depaginate( self.meta2_index_fetch, volume_id=volume_id, listing_key=lambda x: x['records'], truncated_key=lambda x: x['truncated'], # The following is only called when the list is truncated # So we can assume there are records in the list marker_key=lambda x: x['records'][-1]['container_url'], **kwargs)
def process_container(self, container, **kwargs): """ Match then apply the set of rules of the lifecycle configuration on all objects of the container. :returns: tuples of (object metadata, rule name, action, status) :rtype: generator of 4-tuples :notice: the results must be consumed or the rules won't be applied. """ for obj_meta in depaginate(self.api.object_list, listing_key=lambda x: x['objects'], marker_key=lambda x: x.get('next_marker'), truncated_key=lambda x: x['truncated'], account=self.account, container=container, properties=True, versions=True, **kwargs): try: # Save the name of the object as it is in the container, # for later use. obj_meta['orig_name'] = obj_meta['name'] # And reconstruct the name of the object as it is # when shown to the final user. if self.recursive and CH_ENCODED_SEPARATOR in container: obj_meta['name'] = container.split(CH_ENCODED_SEPARATOR, 1)[1] obj_meta['name'].replace(CH_ENCODED_SEPARATOR, CH_SEPARATOR) obj_meta['name'] += CH_SEPARATOR + obj_meta['orig_name'] obj_meta['container'] = container if self.processed_versions is not None \ and self.processed_versions.is_already_processed( obj_meta, **kwargs): continue results = self.apply(obj_meta, **kwargs) for res in results: yield res except Exception as exc: self.logger.warn( "Failed to apply lifecycle rules on %s/%s/%s: %s", self.account, self.container, obj_meta['name'], exc) yield obj_meta, "n/a", "n/a", exc if self.processed_versions is not None: self.processed_versions.save_object(obj_meta, **kwargs)
def _list_autocontainer_objects(self, container, account, **kwargs): object_list = [] if not self.flatns_manager.verify(container): self.log.debug("Container %s is not an autocontainer", container) return object_list self.log.debug("Listing autocontainer %s", container) try: for i in depaginate(self.app.client_manager.storage.object_list, listing_key=lambda x: x['objects'], marker_key=lambda x: x.get('next_marker'), truncated_key=lambda x: x['truncated'], account=account, container=container, **kwargs): object_list.append(i) except exceptions.OioException as err: self.log.warn('Listing may be incomplete: container %s: %s', container, err) return object_list
def take_action(self, parsed_args): self.log.debug('take_action(%s)', parsed_args) super(ListObject, self).take_action(parsed_args) kwargs = {} if parsed_args.prefix: kwargs['prefix'] = parsed_args.prefix if parsed_args.marker: kwargs['marker'] = parsed_args.marker if parsed_args.end_marker: kwargs['end_marker'] = parsed_args.end_marker if parsed_args.delimiter: kwargs['delimiter'] = parsed_args.delimiter if parsed_args.limit and not parsed_args.full_listing: kwargs['limit'] = parsed_args.limit if parsed_args.long_listing: kwargs['properties'] = True if parsed_args.versions: kwargs['versions'] = True if parsed_args.local: kwargs['local'] = True if parsed_args.concurrency: kwargs['concurrency'] = parsed_args.concurrency if parsed_args.attempts: kwargs['request_attempts'] = parsed_args.attempts account = self.app.client_manager.account if parsed_args.auto: obj_gen = self._autocontainer_loop(account, **kwargs) else: container = parsed_args.container if parsed_args.full_listing: obj_gen = depaginate( self.app.client_manager.storage.object_list, listing_key=lambda x: x['objects'], marker_key=lambda x: x.get('next_marker'), truncated_key=lambda x: x['truncated'], account=account, container=container, **kwargs) else: resp = self.app.client_manager.storage.object_list( account, container, **kwargs) obj_gen = resp['objects'] if parsed_args.long_listing: from oio.common.timestamp import Timestamp def _format_props(props): prop_list = ["%s=%s" % (k, v) for k, v in props.items()] if parsed_args.formatter == 'table': prop_string = "\n".join(prop_list) elif parsed_args.formatter in ('value', 'csv'): prop_string = " ".join(prop_list) else: prop_string = props return prop_string def _gen_results(objects): for obj in objects: result = (obj['name'], obj['size'], obj['hash'], obj['version'], obj['deleted'], obj['mime_type'], Timestamp(obj['ctime']).isoformat, obj['policy'], _format_props(obj.get('properties', {}))) yield result results = _gen_results(obj_gen) columns = ('Name', 'Size', 'Hash', 'Version', 'Deleted', 'Content-Type', 'Last-Modified', 'Policy', 'Properties') else: results = ((obj['name'], obj['size'] if not obj['deleted'] else 'deleted', obj['hash'], obj['version']) for obj in obj_gen) columns = ('Name', 'Size', 'Hash', 'Version') return (columns, results)
def take_action(self, parsed_args): self.log.debug('take_action(%s)', parsed_args) kwargs = {'reqid': request_id(prefix='CLI-BUCKET-')} if parsed_args.prefix: kwargs['prefix'] = parsed_args.prefix if parsed_args.marker: kwargs['marker'] = parsed_args.marker if parsed_args.limit: kwargs['limit'] = parsed_args.limit account = self.app.client_manager.account acct_client = self.app.client_manager.storage.account storage = self.app.client_manager.storage if parsed_args.full_listing: listing = depaginate(acct_client.bucket_list, listing_key=lambda x: x['listing'], marker_key=lambda x: x.get('next_marker'), truncated_key=lambda x: x['truncated'], account=account, **kwargs) else: acct_meta = acct_client.bucket_list(account, **kwargs) listing = acct_meta['listing'] columns = ( 'Name', 'Bytes', 'Objects', 'Mtime', ) def versioning(bucket): try: data = storage.container_get_properties(account, bucket, reqid=kwargs['reqid']) except NoSuchContainer: self.log.info('Bucket %s does not exist', bucket) return "Error" sys = data['system'] # WARN it doe not reflect namespace versioning if enabled status = sys.get(M2_PROP_VERSIONING_POLICY, None) if status is None or int(status) == 0: return "Suspended" else: return "Enabled" if parsed_args.versioning: columns += ('Versioning', ) def enrich(listing): for v in listing: v['versioning'] = versioning(v['name']) yield v listing = enrich(listing) return columns, ([v[k.lower()] for k in columns] for v in listing)