def hashes_filter(next_part_power, suff_path, hashes): hashes = list(hashes) for hsh in hashes: fname = os.path.join(suff_path, hsh, 'fake-file-name') if replace_partition_in_path(fname, next_part_power) == fname: hashes.remove(hsh) return hashes
def relink(swift_dir='/etc/swift', devices='/srv/node', skip_mount_check=False, logger=logging.getLogger(), device=None): mount_check = not skip_mount_check run = False relinked = errors = 0 for policy in POLICIES: policy.object_ring = None # Ensure it will be reloaded policy.load_ring(swift_dir) part_power = policy.object_ring.part_power next_part_power = policy.object_ring.next_part_power if not next_part_power or next_part_power == part_power: continue logging.info('Relinking files for policy %s under %s', policy.name, devices) run = True datadir = diskfile.get_data_dir(policy) locks = [None] states = {} relink_devices_filter = partial(devices_filter, device) relink_hook_pre_device = partial(hook_pre_device, locks, states, datadir) relink_hook_post_device = partial(hook_post_device, locks) relink_partition_filter = partial(partitions_filter, states, STEP_RELINK, part_power, next_part_power) relink_hook_post_partition = partial(hook_post_partition, states, STEP_RELINK) relink_hashes_filter = partial(hashes_filter, next_part_power) locations = audit_location_generator( devices, datadir, mount_check=mount_check, devices_filter=relink_devices_filter, hook_pre_device=relink_hook_pre_device, hook_post_device=relink_hook_post_device, partitions_filter=relink_partition_filter, hook_post_partition=relink_hook_post_partition, hashes_filter=relink_hashes_filter) for fname, _, _ in locations: newfname = replace_partition_in_path(fname, next_part_power) try: diskfile.relink_paths(fname, newfname, check_existing=True) relinked += 1 except OSError as exc: errors += 1 logger.warning("Relinking %s to %s failed: %s", fname, newfname, exc) if not run: logger.warning("No policy found to increase the partition power.") return 2 logging.info('Relinked %d diskfiles (%d errors)', relinked, errors) if errors > 0: return 1 return 0
def hashes_filter(self, suff_path, hashes): hashes = list(hashes) for hsh in hashes: fname = os.path.join(suff_path, hsh) if fname == replace_partition_in_path(self.conf['devices'], fname, self.next_part_power): hashes.remove(hsh) return hashes
def process_policy(self, policy): self.logger.info( 'Processing files for policy %s under %s (cleanup=%s)', policy.name, self.root, self.do_cleanup) self.part_power = policy.object_ring.part_power self.next_part_power = policy.object_ring.next_part_power self.diskfile_mgr = self.diskfile_router[policy] self.datadir = diskfile.get_data_dir(policy) self.states = { "part_power": self.part_power, "next_part_power": self.next_part_power, "state": {}, } audit_stats = {} locations = audit_location_generator( self.conf['devices'], self.datadir, mount_check=self.conf['mount_check'], devices_filter=self.devices_filter, hook_pre_device=self.hook_pre_device, hook_post_device=self.hook_post_device, partitions_filter=self.partitions_filter, hook_pre_partition=self.hook_pre_partition, hook_post_partition=self.hook_post_partition, hashes_filter=self.hashes_filter, logger=self.logger, error_counter=audit_stats, yield_hash_dirs=True) if self.conf['files_per_second'] > 0: locations = RateLimitedIterator(locations, self.conf['files_per_second']) for hash_path, device, partition in locations: # note, in cleanup step next_part_power == part_power new_hash_path = replace_partition_in_path(self.conf['devices'], hash_path, self.next_part_power) if new_hash_path == hash_path: continue self.process_location(hash_path, new_hash_path) # any unmounted devices don't trigger the pre_device trigger. # so we'll deal with them here. for dev in audit_stats.get('unmounted', []): self.place_policy_stat(dev, policy, 'unmounted', 1) # Further unlistable_partitions doesn't trigger the post_device, so # we also need to deal with them here. for datadir in audit_stats.get('unlistable_partitions', []): device_path, _ = os.path.split(datadir) device = os.path.basename(device_path) self.place_policy_stat(device, policy, 'unlistable_partitions', 1)
def check_existing(self, new_file): existing_link = None link_created = False start = self.next_part_power - 1 stop = max(start - self.conf['link_check_limit'], -1) for check_part_power in range(start, stop, -1): # Try to create the link from each of several previous part power # locations. If an attempt succeeds then either a link was made or # an existing link with the same inode as the next part power # location was found: either is acceptable. The part power location # that previously failed with EEXIST is included in the further # attempts here for simplicity. target_file = replace_partition_in_path( self.conf['devices'], new_file, check_part_power) try: link_created = diskfile.relink_paths(target_file, new_file, ignore_missing=False) existing_link = target_file break except OSError: pass return existing_link, link_created
def process_policy(self, policy): self.logger.info( 'Processing files for policy %s under %s (cleanup=%s)', policy.name, self.root, self.do_cleanup) self.part_power = policy.object_ring.part_power self.next_part_power = policy.object_ring.next_part_power self.diskfile_mgr = self.diskfile_router[policy] self.datadir = diskfile.get_data_dir(policy) self.states = { "part_power": self.part_power, "next_part_power": self.next_part_power, "state": {}, } locations = audit_location_generator( self.conf['devices'], self.datadir, mount_check=self.conf['mount_check'], devices_filter=self.devices_filter, hook_pre_device=self.hook_pre_device, hook_post_device=self.hook_post_device, partitions_filter=self.partitions_filter, hook_post_partition=self.hook_post_partition, hashes_filter=self.hashes_filter, logger=self.logger, error_counter=self.stats, yield_hash_dirs=True ) if self.conf['files_per_second'] > 0: locations = RateLimitedIterator( locations, self.conf['files_per_second']) for hash_path, device, partition in locations: # note, in cleanup step next_part_power == part_power new_hash_path = replace_partition_in_path( self.conf['devices'], hash_path, self.next_part_power) if new_hash_path == hash_path: continue self.process_location(hash_path, new_hash_path)
def relink(swift_dir='/etc/swift', devices='/srv/node', skip_mount_check=False, logger=logging.getLogger()): mount_check = not skip_mount_check run = False relinked = errors = 0 for policy in POLICIES: policy.object_ring = None # Ensure it will be reloaded policy.load_ring(swift_dir) part_power = policy.object_ring.part_power next_part_power = policy.object_ring.next_part_power if not next_part_power or next_part_power == part_power: continue logging.info('Relinking files for policy %s under %s', policy.name, devices) run = True locations = audit_location_generator( devices, diskfile.get_data_dir(policy), mount_check=mount_check) for fname, _, _ in locations: newfname = replace_partition_in_path(fname, next_part_power) try: diskfile.relink_paths(fname, newfname, check_existing=True) relinked += 1 except OSError as exc: errors += 1 logger.warning("Relinking %s to %s failed: %s", fname, newfname, exc) if not run: logger.warning("No policy found to increase the partition power.") return 2 logging.info('Relinked %d diskfiles (%d errors)', relinked, errors) if errors > 0: return 1 return 0
def cleanup(conf, logger, device): diskfile_router = diskfile.DiskFileRouter(conf, logger) errors = cleaned_up = 0 error_counter = {} found_policy = False for policy in POLICIES: diskfile_mgr = diskfile_router[policy] policy.object_ring = None # Ensure it will be reloaded policy.load_ring(conf['swift_dir']) part_power = policy.object_ring.part_power next_part_power = policy.object_ring.next_part_power if not next_part_power or next_part_power != part_power: continue logger.info('Cleaning up files for policy %s under %s', policy.name, conf['devices']) found_policy = True datadir = diskfile.get_data_dir(policy) locks = [None] states = { "part_power": part_power, "next_part_power": next_part_power, "state": {}, } cleanup_devices_filter = partial(devices_filter, device) cleanup_hook_pre_device = partial(hook_pre_device, locks, states, datadir) cleanup_hook_post_device = partial(hook_post_device, locks) cleanup_partition_filter = partial(partitions_filter, states, part_power, next_part_power) cleanup_hook_post_partition = partial(hook_post_partition, states, STEP_CLEANUP, policy, diskfile_mgr) cleanup_hashes_filter = partial(hashes_filter, next_part_power) locations = audit_location_generator( conf['devices'], datadir, mount_check=conf['mount_check'], devices_filter=cleanup_devices_filter, hook_pre_device=cleanup_hook_pre_device, hook_post_device=cleanup_hook_post_device, partitions_filter=cleanup_partition_filter, hook_post_partition=cleanup_hook_post_partition, hashes_filter=cleanup_hashes_filter, logger=logger, error_counter=error_counter) if conf['files_per_second'] > 0: locations = RateLimitedIterator(locations, conf['files_per_second']) for fname, device, partition in locations: expected_fname = replace_partition_in_path(fname, part_power) if fname == expected_fname: continue # Make sure there is a valid object file in the expected new # location. Note that this could be newer than the original one # (which happens if there is another PUT after partition power # has been increased, but cleanup did not yet run) loc = diskfile.AuditLocation(os.path.dirname(expected_fname), device, partition, policy) df = diskfile_mgr.get_diskfile_from_audit_location(loc) try: with df.open(): pass except DiskFileQuarantined as exc: logger.warning( 'ERROR Object %(obj)s failed audit and was' ' quarantined: %(err)r', { 'obj': loc, 'err': exc }) errors += 1 continue except DiskFileDeleted: pass except DiskFileNotExist as exc: err = False if policy.policy_type == 'erasure_coding': # Might be a non-durable fragment - check that there is # a fragment in the new path. Will be fixed by the # reconstructor then if not os.path.isfile(expected_fname): err = True else: err = True if err: logger.warning('Error cleaning up %s: %r', fname, exc) errors += 1 continue try: os.remove(fname) cleaned_up += 1 logger.debug("Removed %s", fname) suffix_dir = os.path.dirname(os.path.dirname(fname)) diskfile.invalidate_hash(suffix_dir) except OSError as exc: logger.warning('Error cleaning up %s: %r', fname, exc) errors += 1 return determine_exit_code( logger=logger, found_policy=found_policy, processed=cleaned_up, action='cleaned up', action_errors=errors, error_counter=error_counter, )
def relink(conf, logger, device): diskfile_router = diskfile.DiskFileRouter(conf, logger) found_policy = False relinked = errors = 0 error_counter = {} for policy in POLICIES: diskfile_mgr = diskfile_router[policy] policy.object_ring = None # Ensure it will be reloaded policy.load_ring(conf['swift_dir']) part_power = policy.object_ring.part_power next_part_power = policy.object_ring.next_part_power if not next_part_power or next_part_power == part_power: continue logger.info('Relinking files for policy %s under %s', policy.name, conf['devices']) found_policy = True datadir = diskfile.get_data_dir(policy) locks = [None] states = { "part_power": part_power, "next_part_power": next_part_power, "state": {}, } relink_devices_filter = partial(devices_filter, device) relink_hook_pre_device = partial(hook_pre_device, locks, states, datadir) relink_hook_post_device = partial(hook_post_device, locks) relink_partition_filter = partial(partitions_filter, states, part_power, next_part_power) relink_hook_post_partition = partial(hook_post_partition, states, STEP_RELINK, policy, diskfile_mgr) relink_hashes_filter = partial(hashes_filter, next_part_power) locations = audit_location_generator( conf['devices'], datadir, mount_check=conf['mount_check'], devices_filter=relink_devices_filter, hook_pre_device=relink_hook_pre_device, hook_post_device=relink_hook_post_device, partitions_filter=relink_partition_filter, hook_post_partition=relink_hook_post_partition, hashes_filter=relink_hashes_filter, logger=logger, error_counter=error_counter) if conf['files_per_second'] > 0: locations = RateLimitedIterator(locations, conf['files_per_second']) for fname, _, _ in locations: newfname = replace_partition_in_path(fname, next_part_power) try: diskfile.relink_paths(fname, newfname, check_existing=True) relinked += 1 suffix_dir = os.path.dirname(os.path.dirname(newfname)) diskfile.invalidate_hash(suffix_dir) except OSError as exc: errors += 1 logger.warning("Relinking %s to %s failed: %s", fname, newfname, exc) return determine_exit_code( logger=logger, found_policy=found_policy, processed=relinked, action='relinked', action_errors=errors, error_counter=error_counter, )
def _test_main(self, cancel=False): container = 'container-%s' % uuid4() obj = 'object-%s' % uuid4() obj2 = 'object-%s' % uuid4() # Create container headers = {'X-Storage-Policy': self.policy.name} client.put_container(self.url, self.token, container, headers=headers) # Create a new object client.put_object(self.url, self.token, container, obj, self.data) client.head_object(self.url, self.token, container, obj) # Prepare partition power increase builder = RingBuilder.load(self.builder_file) builder.prepare_increase_partition_power() builder.save(self.builder_file) ring_data = builder.get_ring() ring_data.save(self.ring_file) # Ensure the proxy uses the changed ring Manager(['proxy']).restart() # Ensure object is still accessible client.head_object(self.url, self.token, container, obj) # Relink existing objects for device in self.devices: self.assertEqual(0, relink(skip_mount_check=True, devices=device)) # Create second object after relinking and ensure it is accessible client.put_object(self.url, self.token, container, obj2, self.data) client.head_object(self.url, self.token, container, obj2) # Remember the original object locations org_locations = self._find_objs_ondisk(container, obj) org_locations += self._find_objs_ondisk(container, obj2) # Remember the new object locations new_locations = [] for loc in org_locations: new_locations.append( replace_partition_in_path(str(loc), self.object_ring.part_power + 1)) # Overwrite existing object - to ensure that older timestamp files # will be cleaned up properly later client.put_object(self.url, self.token, container, obj, self.data) # Ensure objects are still accessible client.head_object(self.url, self.token, container, obj) client.head_object(self.url, self.token, container, obj2) # Increase partition power builder = RingBuilder.load(self.builder_file) if not cancel: builder.increase_partition_power() else: builder.cancel_increase_partition_power() builder.save(self.builder_file) ring_data = builder.get_ring() ring_data.save(self.ring_file) # Ensure the proxy uses the changed ring Manager(['proxy']).restart() # Ensure objects are still accessible client.head_object(self.url, self.token, container, obj) client.head_object(self.url, self.token, container, obj2) # Overwrite existing object - to ensure that older timestamp files # will be cleaned up properly later client.put_object(self.url, self.token, container, obj, self.data) # Cleanup old objects in the wrong location for device in self.devices: self.assertEqual(0, cleanup(skip_mount_check=True, devices=device)) # Ensure objects are still accessible client.head_object(self.url, self.token, container, obj) client.head_object(self.url, self.token, container, obj2) # Ensure data in old or relinked object locations is removed if not cancel: for fn in org_locations: self.assertFalse(os.path.exists(fn)) else: for fn in new_locations: self.assertFalse(os.path.exists(fn))
def process_location(self, hash_path, new_hash_path): # Compare the contents of each hash dir with contents of same hash # dir in its new partition to verify that the new location has the # most up to date set of files. The new location may have newer # files if it has been updated since relinked. self.stats['hash_dirs'] += 1 # Get on disk data for new and old locations, cleaning up any # reclaimable or obsolete files in each. The new location is # cleaned up *before* the old location to prevent false negatives # where the old still has a file that has been cleaned up in the # new; cleaning up the new location first ensures that the old will # always be 'cleaner' than the new. new_df_data = self.diskfile_mgr.cleanup_ondisk_files(new_hash_path) old_df_data = self.diskfile_mgr.cleanup_ondisk_files(hash_path) # Now determine the most up to date set of on disk files would be # given the content of old and new locations... new_files = set(new_df_data['files']) old_files = set(old_df_data['files']) union_files = new_files.union(old_files) union_data = self.diskfile_mgr.get_ondisk_files( union_files, '', verify=False) obsolete_files = set(info['filename'] for info in union_data.get('obsolete', [])) # drop 'obsolete' files but retain 'unexpected' files which might # be misplaced diskfiles from another policy required_files = union_files.difference(obsolete_files) required_links = required_files.intersection(old_files) missing_links = 0 created_links = 0 unwanted_files = [] for filename in required_links: # Before removing old files, be sure that the corresponding # required new files exist by calling relink_paths again. There # are several possible outcomes: # - The common case is that the new file exists, in which case # relink_paths checks that the new file has the same inode # as the old file. An exception is raised if the inode of # the new file is not the same as the old file. # - The new file may not exist because the relinker failed to # create the link to the old file and has erroneously moved # on to cleanup. In this case the relink_paths will create # the link now or raise an exception if that fails. # - The new file may not exist because some other process, # such as an object server handling a request, has cleaned # it up since we called cleanup_ondisk_files(new_hash_path). # In this case a new link will be created to the old file. # This is unnecessary but simpler than repeating the # evaluation of what links are now required and safer than # assuming that a non-existent file that *was* required is # no longer required. The new file will eventually be # cleaned up again. self.stats['files'] += 1 old_file = os.path.join(hash_path, filename) new_file = os.path.join(new_hash_path, filename) try: if diskfile.relink_paths(old_file, new_file): self.logger.debug( "Relinking%s created link: %s to %s", ' (cleanup)' if self.do_cleanup else '', old_file, new_file) created_links += 1 self.stats['linked'] += 1 except OSError as exc: existing_link = None link_created = False if exc.errno == errno.EEXIST and filename.endswith('.ts'): # special case for duplicate tombstones in older partition # power locations # (https://bugs.launchpad.net/swift/+bug/1921718) existing_link, link_created = self.check_existing(new_file) if existing_link: self.logger.debug( "Relinking%s: link not needed: %s to %s due to " "existing %s", ' (cleanup)' if self.do_cleanup else '', old_file, new_file, existing_link) if link_created: # uncommon case: the retry succeeded in creating a link created_links += 1 self.stats['linked'] += 1 wanted_file = replace_partition_in_path( self.conf['devices'], old_file, self.part_power) if old_file not in (existing_link, wanted_file): # A link exists to a different file and this file # is not the current target for client requests. If # this file is visited again it is possible that # the existing_link will have been cleaned up and # the check will fail, so clean it up now. self.logger.info( "Relinking%s: cleaning up unwanted file: %s", ' (cleanup)' if self.do_cleanup else '', old_file) unwanted_files.append(filename) else: self.logger.warning( "Error relinking%s: failed to relink %s to %s: %s", ' (cleanup)' if self.do_cleanup else '', old_file, new_file, exc) self.stats['errors'] += 1 missing_links += 1 if created_links: self.linked_into_partitions.add(get_partition_from_path( self.conf['devices'], new_hash_path)) diskfile.invalidate_hash(os.path.dirname(new_hash_path)) if self.do_cleanup and not missing_links: # use the sorted list to help unit testing unwanted_files = old_df_data['files'] # the new partition hash dir has the most up to date set of on # disk files so it is safe to delete the old location... rehash = False for filename in unwanted_files: old_file = os.path.join(hash_path, filename) try: os.remove(old_file) except OSError as exc: self.logger.warning('Error cleaning up %s: %r', old_file, exc) self.stats['errors'] += 1 else: rehash = True self.stats['removed'] += 1 self.logger.debug("Removed %s", old_file) if rehash: # Even though we're invalidating the suffix, don't update # self.linked_into_partitions -- we only care about them for # relinking into the new part-power space try: diskfile.invalidate_hash(os.path.dirname(hash_path)) except Exception as exc: # note: not counted as an error self.logger.warning( 'Error invalidating suffix for %s: %r', hash_path, exc)
def cleanup(swift_dir='/etc/swift', devices='/srv/node', skip_mount_check=False, logger=logging.getLogger()): mount_check = not skip_mount_check conf = {'devices': devices, 'mount_check': mount_check} diskfile_router = diskfile.DiskFileRouter(conf, get_logger(conf)) errors = cleaned_up = 0 run = False for policy in POLICIES: policy.object_ring = None # Ensure it will be reloaded policy.load_ring(swift_dir) part_power = policy.object_ring.part_power next_part_power = policy.object_ring.next_part_power if not next_part_power or next_part_power != part_power: continue logging.info('Cleaning up files for policy %s under %s', policy.name, devices) run = True locations = audit_location_generator( devices, diskfile.get_data_dir(policy), mount_check=mount_check) for fname, device, partition in locations: expected_fname = replace_partition_in_path(fname, part_power) if fname == expected_fname: continue # Make sure there is a valid object file in the expected new # location. Note that this could be newer than the original one # (which happens if there is another PUT after partition power # has been increased, but cleanup did not yet run) loc = diskfile.AuditLocation( os.path.dirname(expected_fname), device, partition, policy) diskfile_mgr = diskfile_router[policy] df = diskfile_mgr.get_diskfile_from_audit_location(loc) try: with df.open(): pass except DiskFileQuarantined as exc: logger.warning('ERROR Object %(obj)s failed audit and was' ' quarantined: %(err)r', {'obj': loc, 'err': exc}) errors += 1 continue except DiskFileDeleted: pass except DiskFileNotExist as exc: err = False if policy.policy_type == 'erasure_coding': # Might be a non-durable fragment - check that there is # a fragment in the new path. Will be fixed by the # reconstructor then if not os.path.isfile(expected_fname): err = True else: err = True if err: logger.warning( 'Error cleaning up %s: %r', fname, exc) errors += 1 continue try: os.remove(fname) cleaned_up += 1 logging.debug("Removed %s", fname) except OSError as exc: logger.warning('Error cleaning up %s: %r', fname, exc) errors += 1 if not run: logger.warning("No policy found to increase the partition power.") return 2 logging.info('Cleaned up %d diskfiles (%d errors)', cleaned_up, errors) if errors > 0: return 1 return 0