class ContentFactory(object): DEFAULT_DATASEC = "plain", {"nb_copy": "1", "distance": "0"} def __init__(self, conf, **kwargs): self.conf = conf self.logger = get_logger(conf) self.container_client = ContainerClient(conf, **kwargs) def get(self, container_id, content_id): try: meta, chunks = self.container_client.content_locate( cid=container_id, content=content_id) except NotFound: raise ContentNotFound("Content %s/%s not found" % (container_id, content_id)) chunk_method = meta['chunk_method'] storage_method = STORAGE_METHODS.load(chunk_method) cls = ECContent if storage_method.ec else PlainContent return cls(self.conf, container_id, meta, chunks, storage_method, container_client=self.container_client) def new(self, container_id, path, size, policy): meta, chunks = self.container_client.content_prepare(cid=container_id, path=path, size=size, stgpol=policy) chunk_method = meta['chunk_method'] storage_method = STORAGE_METHODS.load(chunk_method) cls = ECContent if storage_method.ec else PlainContent return cls(self.conf, container_id, meta, chunks, storage_method) def change_policy(self, container_id, content_id, new_policy): old_content = self.get(container_id, content_id) if old_content.stgpol == new_policy: return old_content new_content = self.new(container_id, old_content.path, old_content.length, new_policy) stream = old_content.fetch() new_content.create(GeneratorIO(stream)) # the old content is automatically deleted because the new content has # the same name (but not the same id) return new_content
class ContentRepairerWorker(ToolWorker): def __init__(self, tool, queue_workers, queue_reply): super(ContentRepairerWorker, self).__init__(tool, queue_workers, queue_reply) self.chunk_operator = ChunkOperator(self.conf, logger=self.logger) self.blob_client = BlobClient(self.conf) self.container_client = ContainerClient(self.conf, logger=self.logger) def _safe_chunk_rebuild(self, item, content_id, chunk_id_or_pos, **kwargs): _, account, container, _, _ = item try: container_id = cid_from_name(account, container) self.chunk_operator.rebuild(container_id, content_id, chunk_id_or_pos, **kwargs) except Exception as exc: # pylint: disable=broad-except self.logger.error('ERROR when rebuilding chunk %s (%s): %s', self.tool.string_from_item(item), chunk_id_or_pos, exc) return exc def _repair_metachunk(self, item, content_id, stg_met, pos, chunks): """ Check that a metachunk has the right number of chunks. :returns: the list (generator) of missing chunks """ exceptions = list() required = stg_met.expected_chunks if len(chunks) < required: if stg_met.ec: subs = {x['num'] for x in chunks} for sub in range(required): if sub not in subs: exc = self._safe_chunk_rebuild(item, content_id, "%d.%d" % (pos, sub)) if exc: exceptions.append(exc) else: missing_chunks = required - len(chunks) for _ in range(missing_chunks): exc = self._safe_chunk_rebuild(item, content_id, pos) if exc: exceptions.append(exc) for chunk in chunks: try: self.blob_client.chunk_head(chunk['url'], xattr=True, check_hash=True) except (NotFound, ClientPreconditionFailed) as e: kwargs = { 'try_chunk_delete': isinstance(e, ClientPreconditionFailed) } exc = self._safe_chunk_rebuild(item, content_id, chunk['url'], **kwargs) if exc: exceptions.append(exc) except Exception as exc: # pylint: disable=broad-except self.logger.error('ERROR when checking chunk %s (%s): %s', self.tool.string_from_item(item), chunk['url'], exc) exceptions.append(exc) return exceptions def _process_item(self, item): namespace, account, container, obj_name, version = item if namespace != self.tool.namespace: raise ValueError('Invalid namespace (actual=%s, expected=%s)' % (namespace, self.tool.namespace)) obj_meta, chunks = self.container_client.content_locate( account=account, reference=container, path=obj_name, version=version, properties=False) content_id = obj_meta['id'] exceptions = list() stg_met = STORAGE_METHODS.load(obj_meta['chunk_method']) chunks_by_pos = _sort_chunks(chunks, stg_met.ec) for pos, chunks in iteritems(chunks_by_pos): try: exceptions += self._repair_metachunk(item, content_id, stg_met, pos, chunks) except Exception as exc: # pylint: disable=broad-except self.logger.error('ERROR when repair metachunk %s (%d): %s', self.tool.string_from_item(item), pos, exc) exceptions.append(exc) if exceptions: raise Exception(exceptions) self.container_client.content_touch(account=account, reference=container, path=obj_name, version=version)
class ContentFactory(object): DEFAULT_DATASEC = "plain", {"nb_copy": "1", "distance": "0"} def __init__(self, conf, **kwargs): self.conf = conf self.logger = get_logger(conf) self.container_client = ContainerClient(conf, logger=self.logger, **kwargs) def get(self, container_id, content_id, account=None, container_name=None): try: meta, chunks = self.container_client.content_locate( cid=container_id, content=content_id) except NotFound: raise ContentNotFound("Content %s/%s not found" % (container_id, content_id)) chunk_method = meta['chunk_method'] storage_method = STORAGE_METHODS.load(chunk_method) if not account or not container_name: container_info = self.container_client.container_get_properties( cid=container_id)['system'] if not account: account = container_info['sys.account'] if not container_name: container_name = container_info['sys.user.name'] cls = ECContent if storage_method.ec else PlainContent return cls(self.conf, container_id, meta, chunks, storage_method, account, container_name, container_client=self.container_client) def new(self, container_id, path, size, policy, account=None, container_name=None): meta, chunks = self.container_client.content_prepare(cid=container_id, path=path, size=size, stgpol=policy) chunk_method = meta['chunk_method'] storage_method = STORAGE_METHODS.load(chunk_method) if not account or not container_name: container_info = self.container_client.container_get_properties( cid=container_id)['system'] if not account: account = container_info['sys.account'] if not container_name: container_name = container_info['sys.user.name'] cls = ECContent if storage_method.ec else PlainContent return cls(self.conf, container_id, meta, chunks, storage_method, account, container_name) def copy(self, origin, policy=None): if not policy: policy = origin.policy metadata = origin.metadata.copy() new_metadata, chunks = self.container_client.content_prepare( cid=origin.container_id, path=metadata['name'], size=metadata['length'], stgpol=policy) metadata['chunk_method'] = new_metadata['chunk_method'] metadata['chunk_size'] = new_metadata['chunk_size'] # We must use a new content_id since we change the data metadata['id'] = new_metadata['id'] # We may want to keep the same version, but it is denied by meta2 metadata['version'] = int(metadata['version']) + 1 metadata['policy'] = new_metadata['policy'] # FIXME: meta2 does not allow us to set ctime # and thus the object will appear as new. storage_method = STORAGE_METHODS.load(metadata['chunk_method']) cls = ECContent if storage_method.ec else PlainContent return cls(self.conf, origin.container_id, metadata, chunks, storage_method, origin.account, origin.container_name) def change_policy(self, container_id, content_id, new_policy): old_content = self.get(container_id, content_id) if old_content.policy == new_policy: return old_content new_content = self.copy(old_content, policy=new_policy) stream = old_content.fetch() new_content.create(GeneratorIO(stream)) # the old content is automatically deleted because the new content has # the same name (but not the same id) return new_content
class Checker(object): def __init__(self, namespace, concurrency=50, error_file=None, rebuild_file=None, full=True, limit_listings=0, request_attempts=1): self.pool = GreenPool(concurrency) self.error_file = error_file self.full = bool(full) # Optimisation for when we are only checking one object # or one container. # 0 -> do not limit # 1 -> limit account listings (list of containers) # 2 -> limit container listings (list of objects) self.limit_listings = limit_listings if self.error_file: f = open(self.error_file, 'a') self.error_writer = csv.writer(f, delimiter=' ') self.rebuild_file = rebuild_file if self.rebuild_file: fd = open(self.rebuild_file, 'a') self.rebuild_writer = csv.writer(fd, delimiter='|') conf = {'namespace': namespace} self.account_client = AccountClient(conf, max_retries=request_attempts - 1) self.container_client = ContainerClient( conf, max_retries=request_attempts - 1, request_attempts=request_attempts) self.blob_client = BlobClient(conf=conf) self.accounts_checked = 0 self.containers_checked = 0 self.objects_checked = 0 self.chunks_checked = 0 self.account_not_found = 0 self.container_not_found = 0 self.object_not_found = 0 self.chunk_not_found = 0 self.account_exceptions = 0 self.container_exceptions = 0 self.object_exceptions = 0 self.chunk_exceptions = 0 self.list_cache = {} self.running = {} def write_error(self, target, irreparable=False): error = list() if irreparable: error.append('#IRREPARABLE') error.append(target.account) if target.container: error.append(target.container) if target.obj: error.append(target.obj) if target.chunk: error.append(target.chunk) self.error_writer.writerow(error) def write_rebuilder_input(self, target, obj_meta, irreparable=False): ct_meta = self.list_cache[(target.account, target.container)][1] try: cid = ct_meta['system']['sys.name'].split('.', 1)[0] except KeyError: cid = ct_meta['properties']['sys.name'].split('.', 1)[0] error = list() if irreparable: error.append('#IRREPARABLE') error.append(cid) error.append(obj_meta['id']) error.append(target.chunk) self.rebuild_writer.writerow(error) def write_chunk_error(self, target, obj_meta, chunk=None, irreparable=False): if chunk is not None: target = target.copy() target.chunk = chunk if self.error_file: self.write_error(target, irreparable=irreparable) if self.rebuild_file: self.write_rebuilder_input(target, obj_meta, irreparable=irreparable) def _check_chunk_xattr(self, target, obj_meta, xattr_meta): error = False # Composed position -> erasure coding attr_prefix = 'meta' if '.' in obj_meta['pos'] else '' attr_key = attr_prefix + 'chunk_size' if str(obj_meta['size']) != xattr_meta.get(attr_key): print( " Chunk %s '%s' xattr (%s) " "differs from size in meta2 (%s)" % (target, attr_key, xattr_meta.get(attr_key), obj_meta['size'])) error = True attr_key = attr_prefix + 'chunk_hash' if obj_meta['hash'] != xattr_meta.get(attr_key): print( " Chunk %s '%s' xattr (%s) " "differs from hash in meta2 (%s)" % (target, attr_key, xattr_meta.get(attr_key), obj_meta['hash'])) error = True return error def _check_chunk(self, target): chunk = target.chunk obj_listing, obj_meta = self.check_obj(target) error = False if chunk not in obj_listing: print(' Chunk %s missing from object listing' % target) error = True db_meta = dict() else: db_meta = obj_listing[chunk] try: xattr_meta = self.blob_client.chunk_head(chunk, xattr=self.full) except exc.NotFound as e: self.chunk_not_found += 1 error = True print(' Not found chunk "%s": %s' % (target, str(e))) except Exception as e: self.chunk_exceptions += 1 error = True print(' Exception chunk "%s": %s' % (target, str(e))) else: if db_meta and self.full: error = self._check_chunk_xattr(target, db_meta, xattr_meta) self.chunks_checked += 1 return error, obj_meta def check_chunk(self, target): error, obj_meta = self._check_chunk(target) if error: self.write_chunk_error(target, obj_meta) def _check_metachunk(self, target, obj_meta, stg_met, pos, chunks, recurse=False): required = stg_met.expected_chunks chunk_errors = list() if len(chunks) < required: missing_chunks = required - len(chunks) print(' Missing %d chunks at position %s of %s' % (missing_chunks, pos, target)) if stg_met.ec: subs = {x['num'] for x in chunks} for sub in range(required): if sub not in subs: chunk_errors.append( (target, obj_meta, '%d.%d' % (pos, sub))) else: for _ in range(missing_chunks): chunk_errors.append((target, obj_meta, str(pos))) if recurse: for chunk in chunks: t = target.copy() t.chunk = chunk['url'] error, obj_meta = self._check_chunk(t) if error: chunk_errors.append((t, obj_meta)) irreparable = required - len(chunk_errors) < stg_met.min_chunks_to_read for chunk_error in chunk_errors: self.write_chunk_error(*chunk_error, irreparable=irreparable) def _check_obj_policy(self, target, obj_meta, chunks, recurse=False): """ Check that the list of chunks of an object matches the object's storage policy. """ stg_met = STORAGE_METHODS.load(obj_meta['chunk_method']) chunks_by_pos = _sort_chunks(chunks, stg_met.ec) for pos, chunks in chunks_by_pos.iteritems(): self.pool.spawn_n(self._check_metachunk, target.copy(), obj_meta, stg_met, pos, chunks, recurse=recurse) def check_obj(self, target, recurse=False): account = target.account container = target.container obj = target.obj if (account, container, obj) in self.running: self.running[(account, container, obj)].wait() if (account, container, obj) in self.list_cache: return self.list_cache[(account, container, obj)] self.running[(account, container, obj)] = Event() print('Checking object "%s"' % target) container_listing, ct_meta = self.check_container(target) error = False if obj not in container_listing: print(' Object %s missing from container listing' % target) error = True # checksum = None else: # TODO check checksum match # checksum = container_listing[obj]['hash'] pass results = [] meta = dict() try: meta, results = self.container_client.content_locate( account=account, reference=container, path=obj, properties=False) except exc.NotFound as e: self.object_not_found += 1 error = True print(' Not found object "%s": %s' % (target, str(e))) except Exception as e: self.object_exceptions += 1 error = True print(' Exception object "%s": %s' % (target, str(e))) chunk_listing = dict() for chunk in results: chunk_listing[chunk['url']] = chunk if meta: self.list_cache[(account, container, obj)] = (chunk_listing, meta) self.objects_checked += 1 self.running[(account, container, obj)].send(True) del self.running[(account, container, obj)] # Skip the check if we could not locate the object if meta: self._check_obj_policy(target, meta, results, recurse=recurse) if error and self.error_file: self.write_error(target) return chunk_listing, meta def check_container(self, target, recurse=False): account = target.account container = target.container if (account, container) in self.running: self.running[(account, container)].wait() if (account, container) in self.list_cache: return self.list_cache[(account, container)] self.running[(account, container)] = Event() print('Checking container "%s"' % target) account_listing = self.check_account(target) error = False if container not in account_listing: error = True print(' Container %s missing from account listing' % target) marker = None results = [] ct_meta = dict() extra_args = dict() if self.limit_listings > 1 and target.obj: # When we are explicitly checking one object, start the listing # where this object is supposed to be, and list only one object. extra_args['prefix'] = target.obj extra_args['limit'] = 1 while True: try: _, resp = self.container_client.content_list( account=account, reference=container, marker=marker, **extra_args) except exc.NotFound as e: self.container_not_found += 1 error = True print(' Not found container "%s": %s' % (target, str(e))) break except Exception as e: self.container_exceptions += 1 error = True print(' Exception container "%s": %s' % (target, str(e))) break if resp['objects']: marker = resp['objects'][-1]['name'] results.extend(resp['objects']) if self.limit_listings > 1: break else: ct_meta = resp ct_meta.pop('objects') break container_listing = dict() for obj in results: container_listing[obj['name']] = obj if self.limit_listings <= 1: # We just listed the whole container, keep the result in a cache self.containers_checked += 1 self.list_cache[(account, container)] = container_listing, ct_meta self.running[(account, container)].send(True) del self.running[(account, container)] if recurse: for obj in container_listing: t = target.copy() t.obj = obj self.pool.spawn_n(self.check_obj, t, True) if error and self.error_file: self.write_error(target) return container_listing, ct_meta def check_account(self, target, recurse=False): account = target.account if account in self.running: self.running[account].wait() if account in self.list_cache: return self.list_cache[account] self.running[account] = Event() print('Checking account "%s"' % target) error = False marker = None results = [] extra_args = dict() if self.limit_listings > 0 and target.container: # When we are explicitly checking one container, start the listing # where this container is supposed to be, and list only one # container. extra_args['prefix'] = target.container extra_args['limit'] = 1 while True: try: resp = self.account_client.container_list(account, marker=marker, **extra_args) except Exception as e: self.account_exceptions += 1 error = True print(' Exception account "%s": %s' % (target, str(e))) break if resp['listing']: marker = resp['listing'][-1][0] results.extend(resp['listing']) if self.limit_listings > 0: break else: break containers = dict() for e in results: containers[e[0]] = (e[1], e[2]) if self.limit_listings <= 0: # We just listed the whole account, keep the result in a cache self.accounts_checked += 1 self.list_cache[account] = containers self.running[account].send(True) del self.running[account] if recurse: for container in containers: t = target.copy() t.container = container self.pool.spawn_n(self.check_container, t, True) if error and self.error_file: self.write_error(target) return containers def check(self, target): if target.chunk and target.obj and target.container: self.pool.spawn_n(self.check_chunk, target) elif target.obj and target.container: self.pool.spawn_n(self.check_obj, target, True) elif target.container: self.pool.spawn_n(self.check_container, target, True) else: self.pool.spawn_n(self.check_account, target, True) def wait(self): self.pool.waitall() def report(self): success = True def _report_stat(name, stat): print("{0:18}: {1}".format(name, stat)) print() print('Report') _report_stat("Accounts checked", self.accounts_checked) if self.account_not_found: success = False _report_stat("Missing accounts", self.account_not_found) if self.account_exceptions: success = False _report_stat("Exceptions", self.account_exceptions) print() _report_stat("Containers checked", self.containers_checked) if self.container_not_found: success = False _report_stat("Missing containers", self.container_not_found) if self.container_exceptions: success = False _report_stat("Exceptions", self.container_exceptions) print() _report_stat("Objects checked", self.objects_checked) if self.object_not_found: success = False _report_stat("Missing objects", self.object_not_found) if self.object_exceptions: success = False _report_stat("Exceptions", self.object_exceptions) print() _report_stat("Chunks checked", self.chunks_checked) if self.chunk_not_found: success = False _report_stat("Missing chunks", self.chunk_not_found) if self.chunk_exceptions: success = False _report_stat("Exceptions", self.chunk_exceptions) return success