def __init__(self, conf, logger, rcache, devices, zero_byte_only_at_fps=0):
self.conf = conf
self.logger = logger
self.devices = devices
self.diskfile_router = diskfile.DiskFileRouter(conf, self.logger)
self.max_files_per_second = float(conf.get('files_per_second', 20))
self.max_bytes_per_second = float(
conf.get('bytes_per_second', 10000000))
self.auditor_type = 'ALL'
self.zero_byte_only_at_fps = zero_byte_only_at_fps
if self.zero_byte_only_at_fps:
self.max_files_per_second = float(self.zero_byte_only_at_fps)
self.auditor_type = 'ZBF'
self.log_time = int(conf.get('log_time', 3600))
self.last_logged = 0
self.files_running_time = 0
self.bytes_running_time = 0
self.bytes_processed = 0
self.total_bytes_processed = 0
self.total_files_processed = 0
self.passes = 0
self.quarantines = 0
self.errors = 0
self.rcache = rcache
self.stats_sizes = sorted(
[int(s) for s in list_from_csv(conf.get('object_size_stats'))])
self.stats_buckets = dict([(s, 0)
for s in self.stats_sizes + ['OVER']])
def __init__(self, conf, logger, rcache, devices, zero_byte_only_at_fps=0):
self.conf = conf
self.logger = logger
self.devices = devices
self.max_files_per_second = float(conf.get('files_per_second', 20))
self.max_bytes_per_second = float(
conf.get('bytes_per_second', 10000000))
try:
# ideally unless ops overrides the rsync_tempfile_timeout in the
# auditor section we can base our behavior on whatever they
# configure for their replicator
replicator_config = readconf(self.conf['__file__'],
'object-replicator')
except (KeyError, ValueError, IOError):
# if we can't parse the real config (generally a KeyError on
# __file__, or ValueError on no object-replicator section, or
# IOError if reading the file failed) we use
# a very conservative default for rsync_timeout
default_rsync_timeout = 86400
else:
replicator_rsync_timeout = int(
replicator_config.get('rsync_timeout',
replicator.DEFAULT_RSYNC_TIMEOUT))
# Here we can do some light math for ops and use the *replicator's*
# rsync_timeout (plus 15 mins to avoid deleting local tempfiles
# before the remote replicator kills it's rsync)
default_rsync_timeout = replicator_rsync_timeout + 900
# there's not really a good reason to assume the replicator
# section's reclaim_age is more appropriate than the reconstructor
# reclaim_age - but we're already parsing the config so we can set
# the default value in our config if it's not already set
if 'reclaim_age' in replicator_config:
conf.setdefault('reclaim_age',
replicator_config['reclaim_age'])
self.rsync_tempfile_timeout = config_auto_int_value(
self.conf.get('rsync_tempfile_timeout'), default_rsync_timeout)
self.diskfile_router = diskfile.DiskFileRouter(conf, self.logger)
self.auditor_type = 'ALL'
self.zero_byte_only_at_fps = zero_byte_only_at_fps
if self.zero_byte_only_at_fps:
self.max_files_per_second = float(self.zero_byte_only_at_fps)
self.auditor_type = 'ZBF'
self.log_time = int(conf.get('log_time', 3600))
self.last_logged = 0
self.files_running_time = 0
self.bytes_running_time = 0
self.bytes_processed = 0
self.total_bytes_processed = 0
self.total_files_processed = 0
self.passes = 0
self.quarantines = 0
self.errors = 0
self.rcache = rcache
self.stats_sizes = sorted(
[int(s) for s in list_from_csv(conf.get('object_size_stats'))])
self.stats_buckets = dict([(s, 0)
for s in self.stats_sizes + ['OVER']])
def __init__(self, conf, logger, device_list=None, do_cleanup=False):
self.conf = conf
self.logger = logger
self.device_list = device_list or []
self.do_cleanup = do_cleanup
self.root = self.conf['devices']
if len(self.device_list) == 1:
self.root = os.path.join(self.root, list(self.device_list)[0])
self.part_power = self.next_part_power = None
self.diskfile_mgr = None
self.dev_lock = None
self.diskfile_router = diskfile.DiskFileRouter(self.conf, self.logger)
self._zero_stats()
def __init__(self, testdir, policy=None):
self.logger = debug_logger('test-ssync-sender')
self.conn_timeout = 1
self.node_timeout = 2
self.http_timeout = 3
self.network_chunk_size = 65536
self.disk_chunk_size = 4096
conf = {
'devices': testdir,
'mount_check': 'false',
}
policy = POLICIES.default if policy is None else policy
self._diskfile_router = diskfile.DiskFileRouter(conf, self.logger)
self._diskfile_mgr = self._diskfile_router[policy]
def __init__(self, conf, logger, device_list=None, do_cleanup=False):
self.conf = conf
self.recon_cache = os.path.join(self.conf['recon_cache_path'],
RECON_RELINKER_FILE)
self.logger = logger
self.device_list = device_list or []
self.do_cleanup = do_cleanup
self.root = self.conf['devices']
if len(self.device_list) == 1:
self.root = os.path.join(self.root, list(self.device_list)[0])
self.part_power = self.next_part_power = None
self.diskfile_mgr = None
self.dev_lock = None
self._last_recon_update = time.time()
self.stats_interval = float(
conf.get('stats_interval', DEFAULT_STATS_INTERVAL))
self.diskfile_router = diskfile.DiskFileRouter(self.conf, self.logger)
self.stats = _zero_stats()
self.devices_data = recursive_defaultdict()
self.policy_count = 0
self.pid = os.getpid()
self.linked_into_partitions = set()
def _test_ondisk_data_after_write_with_crypto(self, policy_name):
policy = storage_policy.POLICIES.get_by_name(policy_name)
self._create_container(self.proxy_app, policy_name=policy_name)
self._put_object(self.crypto_app, self.plaintext)
self._post_object(self.crypto_app)
# Verify container listing etag is encrypted by direct GET to container
# server. We can use any server for all nodes since they all share same
# devices dir.
cont_server = self._test_context['test_servers'][3]
cont_ring = Ring(self._test_context['testdir'], ring_name='container')
part, nodes = cont_ring.get_nodes('a', self.container_name)
for node in nodes:
req = Request.blank('/%s/%s/a/%s' %
(node['device'], part, self.container_name),
method='GET',
query_string='format=json')
resp = req.get_response(cont_server)
listing = json.loads(resp.body)
# sanity checks...
self.assertEqual(1, len(listing))
self.assertEqual('o', listing[0]['name'])
self.assertEqual('application/test', listing[0]['content_type'])
# verify encrypted etag value
parts = listing[0]['hash'].rsplit(';', 1)
crypto_meta_param = parts[1].strip()
crypto_meta = crypto_meta_param[len('swift_meta='):]
listing_etag_iv = load_crypto_meta(crypto_meta)['iv']
exp_enc_listing_etag = base64.b64encode(
encrypt(self.plaintext_etag.encode('ascii'),
self.km.create_key('/a/%s' % self.container_name),
listing_etag_iv)).decode('ascii')
self.assertEqual(exp_enc_listing_etag, parts[0])
# Verify diskfile data and metadata is encrypted
ring_object = self.proxy_app.get_object_ring(int(policy))
partition, nodes = ring_object.get_nodes('a', self.container_name, 'o')
conf = {
'devices': self._test_context["testdir"],
'mount_check': 'false'
}
df_mgr = diskfile.DiskFileRouter(conf, FakeLogger())[policy]
ondisk_data = []
exp_enc_body = None
for node_index, node in enumerate(nodes):
df = df_mgr.get_diskfile(node['device'],
partition,
'a',
self.container_name,
'o',
policy=policy)
with df.open():
meta = df.get_metadata()
contents = b''.join(df.reader())
metadata = dict((k.lower(), v) for k, v in meta.items())
# verify on disk data - body
body_iv = load_crypto_meta(
metadata['x-object-sysmeta-crypto-body-meta'])['iv']
body_key_meta = load_crypto_meta(
metadata['x-object-sysmeta-crypto-body-meta'])['body_key']
obj_key = self.km.create_key('/a/%s/o' % self.container_name)
body_key = Crypto().unwrap_key(obj_key, body_key_meta)
exp_enc_body = encrypt(self.plaintext, body_key, body_iv)
ondisk_data.append((node, contents))
# verify on disk user metadata
enc_val, meta = metadata[
'x-object-transient-sysmeta-crypto-meta-fruit'].split(';')
meta = meta.strip()[len('swift_meta='):]
metadata_iv = load_crypto_meta(meta)['iv']
exp_enc_meta = base64.b64encode(
encrypt(b'Kiwi', obj_key, metadata_iv)).decode('ascii')
self.assertEqual(exp_enc_meta, enc_val)
self.assertNotIn('x-object-meta-fruit', metadata)
self.assertIn('x-object-transient-sysmeta-crypto-meta',
metadata)
meta = load_crypto_meta(
metadata['x-object-transient-sysmeta-crypto-meta'])
self.assertIn('key_id', meta)
self.assertIn('path', meta['key_id'])
self.assertEqual(
'/a/%s/%s' % (self.container_name, self.object_name),
meta['key_id']['path'])
self.assertIn('v', meta['key_id'])
self.assertEqual('2', meta['key_id']['v'])
self.assertIn('cipher', meta)
self.assertEqual(Crypto.cipher, meta['cipher'])
# verify etag
actual_enc_etag, _junk, actual_etag_meta = metadata[
'x-object-sysmeta-crypto-etag'].partition('; swift_meta=')
etag_iv = load_crypto_meta(actual_etag_meta)['iv']
exp_enc_etag = base64.b64encode(
encrypt(self.plaintext_etag.encode('ascii'), obj_key,
etag_iv)).decode('ascii')
self.assertEqual(exp_enc_etag, actual_enc_etag)
# verify etag hmac
exp_etag_mac = hmac.new(obj_key,
self.plaintext_etag.encode('ascii'),
digestmod=hashlib.sha256).digest()
exp_etag_mac = base64.b64encode(exp_etag_mac).decode('ascii')
self.assertEqual(exp_etag_mac,
metadata['x-object-sysmeta-crypto-etag-mac'])
# verify etag override for container updates
override = 'x-object-sysmeta-container-update-override-etag'
parts = metadata[override].rsplit(';', 1)
crypto_meta_param = parts[1].strip()
crypto_meta = crypto_meta_param[len('swift_meta='):]
listing_etag_iv = load_crypto_meta(crypto_meta)['iv']
cont_key = self.km.create_key('/a/%s' % self.container_name)
exp_enc_listing_etag = base64.b64encode(
encrypt(self.plaintext_etag.encode('ascii'), cont_key,
listing_etag_iv)).decode('ascii')
self.assertEqual(exp_enc_listing_etag, parts[0])
self._check_GET_and_HEAD(self.crypto_app)
return exp_enc_body, ondisk_data
def cleanup(conf, logger, device):
diskfile_router = diskfile.DiskFileRouter(conf, logger)
errors = cleaned_up = 0
error_counter = {}
found_policy = False
for policy in POLICIES:
diskfile_mgr = diskfile_router[policy]
policy.object_ring = None # Ensure it will be reloaded
policy.load_ring(conf['swift_dir'])
part_power = policy.object_ring.part_power
next_part_power = policy.object_ring.next_part_power
if not next_part_power or next_part_power != part_power:
continue
logger.info('Cleaning up files for policy %s under %s', policy.name,
conf['devices'])
found_policy = True
datadir = diskfile.get_data_dir(policy)
locks = [None]
states = {
"part_power": part_power,
"next_part_power": next_part_power,
"state": {},
}
cleanup_devices_filter = partial(devices_filter, device)
cleanup_hook_pre_device = partial(hook_pre_device, locks, states,
datadir)
cleanup_hook_post_device = partial(hook_post_device, locks)
cleanup_partition_filter = partial(partitions_filter, states,
part_power, next_part_power)
cleanup_hook_post_partition = partial(hook_post_partition, states,
STEP_CLEANUP, policy,
diskfile_mgr)
cleanup_hashes_filter = partial(hashes_filter, next_part_power)
locations = audit_location_generator(
conf['devices'],
datadir,
mount_check=conf['mount_check'],
devices_filter=cleanup_devices_filter,
hook_pre_device=cleanup_hook_pre_device,
hook_post_device=cleanup_hook_post_device,
partitions_filter=cleanup_partition_filter,
hook_post_partition=cleanup_hook_post_partition,
hashes_filter=cleanup_hashes_filter,
logger=logger,
error_counter=error_counter)
if conf['files_per_second'] > 0:
locations = RateLimitedIterator(locations,
conf['files_per_second'])
for fname, device, partition in locations:
expected_fname = replace_partition_in_path(fname, part_power)
if fname == expected_fname:
continue
# Make sure there is a valid object file in the expected new
# location. Note that this could be newer than the original one
# (which happens if there is another PUT after partition power
# has been increased, but cleanup did not yet run)
loc = diskfile.AuditLocation(os.path.dirname(expected_fname),
device, partition, policy)
df = diskfile_mgr.get_diskfile_from_audit_location(loc)
try:
with df.open():
pass
except DiskFileQuarantined as exc:
logger.warning(
'ERROR Object %(obj)s failed audit and was'
' quarantined: %(err)r', {
'obj': loc,
'err': exc
})
errors += 1
continue
except DiskFileDeleted:
pass
except DiskFileNotExist as exc:
err = False
if policy.policy_type == 'erasure_coding':
# Might be a non-durable fragment - check that there is
# a fragment in the new path. Will be fixed by the
# reconstructor then
if not os.path.isfile(expected_fname):
err = True
else:
err = True
if err:
logger.warning('Error cleaning up %s: %r', fname, exc)
errors += 1
continue
try:
os.remove(fname)
cleaned_up += 1
logger.debug("Removed %s", fname)
suffix_dir = os.path.dirname(os.path.dirname(fname))
diskfile.invalidate_hash(suffix_dir)
except OSError as exc:
logger.warning('Error cleaning up %s: %r', fname, exc)
errors += 1
return determine_exit_code(
logger=logger,
found_policy=found_policy,
processed=cleaned_up,
action='cleaned up',
action_errors=errors,
error_counter=error_counter,
)
def relink(conf, logger, device):
diskfile_router = diskfile.DiskFileRouter(conf, logger)
found_policy = False
relinked = errors = 0
error_counter = {}
for policy in POLICIES:
diskfile_mgr = diskfile_router[policy]
policy.object_ring = None # Ensure it will be reloaded
policy.load_ring(conf['swift_dir'])
part_power = policy.object_ring.part_power
next_part_power = policy.object_ring.next_part_power
if not next_part_power or next_part_power == part_power:
continue
logger.info('Relinking files for policy %s under %s', policy.name,
conf['devices'])
found_policy = True
datadir = diskfile.get_data_dir(policy)
locks = [None]
states = {
"part_power": part_power,
"next_part_power": next_part_power,
"state": {},
}
relink_devices_filter = partial(devices_filter, device)
relink_hook_pre_device = partial(hook_pre_device, locks, states,
datadir)
relink_hook_post_device = partial(hook_post_device, locks)
relink_partition_filter = partial(partitions_filter, states,
part_power, next_part_power)
relink_hook_post_partition = partial(hook_post_partition, states,
STEP_RELINK, policy, diskfile_mgr)
relink_hashes_filter = partial(hashes_filter, next_part_power)
locations = audit_location_generator(
conf['devices'],
datadir,
mount_check=conf['mount_check'],
devices_filter=relink_devices_filter,
hook_pre_device=relink_hook_pre_device,
hook_post_device=relink_hook_post_device,
partitions_filter=relink_partition_filter,
hook_post_partition=relink_hook_post_partition,
hashes_filter=relink_hashes_filter,
logger=logger,
error_counter=error_counter)
if conf['files_per_second'] > 0:
locations = RateLimitedIterator(locations,
conf['files_per_second'])
for fname, _, _ in locations:
newfname = replace_partition_in_path(fname, next_part_power)
try:
diskfile.relink_paths(fname, newfname, check_existing=True)
relinked += 1
suffix_dir = os.path.dirname(os.path.dirname(newfname))
diskfile.invalidate_hash(suffix_dir)
except OSError as exc:
errors += 1
logger.warning("Relinking %s to %s failed: %s", fname,
newfname, exc)
return determine_exit_code(
logger=logger,
found_policy=found_policy,
processed=relinked,
action='relinked',
action_errors=errors,
error_counter=error_counter,
)
def cleanup(swift_dir='/etc/swift',
devices='/srv/node',
skip_mount_check=False,
logger=logging.getLogger()):
mount_check = not skip_mount_check
conf = {'devices': devices, 'mount_check': mount_check}
diskfile_router = diskfile.DiskFileRouter(conf, get_logger(conf))
errors = cleaned_up = 0
run = False
for policy in POLICIES:
policy.object_ring = None # Ensure it will be reloaded
policy.load_ring(swift_dir)
part_power = policy.object_ring.part_power
next_part_power = policy.object_ring.next_part_power
if not next_part_power or next_part_power != part_power:
continue
logging.info('Cleaning up files for policy %s under %s',
policy.name, devices)
run = True
locations = audit_location_generator(
devices,
diskfile.get_data_dir(policy),
mount_check=mount_check)
for fname, device, partition in locations:
expected_fname = replace_partition_in_path(fname, part_power)
if fname == expected_fname:
continue
# Make sure there is a valid object file in the expected new
# location. Note that this could be newer than the original one
# (which happens if there is another PUT after partition power
# has been increased, but cleanup did not yet run)
loc = diskfile.AuditLocation(
os.path.dirname(expected_fname), device, partition, policy)
diskfile_mgr = diskfile_router[policy]
df = diskfile_mgr.get_diskfile_from_audit_location(loc)
try:
with df.open():
pass
except DiskFileQuarantined as exc:
logger.warning('ERROR Object %(obj)s failed audit and was'
' quarantined: %(err)r',
{'obj': loc, 'err': exc})
errors += 1
continue
except DiskFileDeleted:
pass
except DiskFileNotExist as exc:
err = False
if policy.policy_type == 'erasure_coding':
# Might be a non-durable fragment - check that there is
# a fragment in the new path. Will be fixed by the
# reconstructor then
if not os.path.isfile(expected_fname):
err = True
else:
err = True
if err:
logger.warning(
'Error cleaning up %s: %r', fname, exc)
errors += 1
continue
try:
os.remove(fname)
cleaned_up += 1
logging.debug("Removed %s", fname)
except OSError as exc:
logger.warning('Error cleaning up %s: %r', fname, exc)
errors += 1
if not run:
logger.warning("No policy found to increase the partition power.")
return 2
logging.info('Cleaned up %d diskfiles (%d errors)', cleaned_up, errors)
if errors > 0:
return 1
return 0
