def test_get_policy_string(self):
self.assertEquals(get_policy_string('something', 0), 'something')
self.assertEquals(get_policy_string('something', None), 'something')
self.assertEquals(get_policy_string('something', ''), 'something')
self.assertEquals(get_policy_string('something', 1),
'something' + '-1')
self.assertRaises(PolicyError, get_policy_string, 'something', 99)
def test_get_policy_string(self):
self.assertEqual(get_policy_string('something', 0), 'something')
self.assertEqual(get_policy_string('something', None), 'something')
self.assertEqual(get_policy_string('something', ''), 'something')
self.assertEqual(get_policy_string('something', 1),
'something' + '-1')
self.assertRaises(PolicyError, get_policy_string, 'something', 99)
def test_split_policy_string(self):
expectations = {
'something': ('something', POLICIES[0]),
'something-1': ('something', POLICIES[1]),
'tmp': ('tmp', POLICIES[0]),
'objects': ('objects', POLICIES[0]),
'tmp-1': ('tmp', POLICIES[1]),
'objects-1': ('objects', POLICIES[1]),
'objects-': PolicyError,
'objects-0': PolicyError,
'objects--1': ('objects-', POLICIES[1]),
'objects-+1': PolicyError,
'objects--': PolicyError,
'objects-foo': PolicyError,
'objects--bar': PolicyError,
'objects-+bar': PolicyError,
# questionable, demonstrated as inverse of get_policy_string
'objects+0': ('objects+0', POLICIES[0]),
'': ('', POLICIES[0]),
'0': ('0', POLICIES[0]),
'-1': ('', POLICIES[1]),
}
for policy_string, expected in expectations.items():
if expected == PolicyError:
try:
invalid = split_policy_string(policy_string)
except PolicyError:
continue # good
else:
self.fail('The string %r returned %r '
'instead of raising a PolicyError' %
(policy_string, invalid))
self.assertEqual(expected, split_policy_string(policy_string))
# should be inverse of get_policy_string
self.assertEqual(policy_string, get_policy_string(*expected))
def iter_all_objects(self, conn, policy):
prefix = get_policy_string('objects', policy)
key_range = [prefix + term for term in ('.', '/')]
last_key, last_key_info = None, {}
for key in conn.iterKeyRange(*key_range, reverse=True):
key_info = split_key(key)
if key_info['ext'] == 'ts' and Timestamp(
key_info['timestamp']) < (
time.time() - self.reclaim_age):
self.logger.debug('reclaiming tombstone %r' % key)
conn.delete(key, force=True).wait()
continue
if not last_key:
last_key, last_key_info = key, key_info
continue
if last_key_info['hashpath'] == key_info['hashpath']:
# this next key is better, we should clean up the old one!
last_key_info['timestamp'] = '0'
temp_marker = temp_key(**last_key_info)
conn.put(temp_marker, '', force=True).wait()
conn.delete(last_key, force=True).wait()
else:
yield last_key
last_key, last_key_info = key, key_info
yield last_key
def test_split_policy_string(self):
expectations = {
'something': ('something', POLICIES[0]),
'something-1': ('something', POLICIES[1]),
'tmp': ('tmp', POLICIES[0]),
'objects': ('objects', POLICIES[0]),
'tmp-1': ('tmp', POLICIES[1]),
'objects-1': ('objects', POLICIES[1]),
'objects-': PolicyError,
'objects-0': PolicyError,
'objects--1': ('objects-', POLICIES[1]),
'objects-+1': PolicyError,
'objects--': PolicyError,
'objects-foo': PolicyError,
'objects--bar': PolicyError,
'objects-+bar': PolicyError,
# questionable, demonstrated as inverse of get_policy_string
'objects+0': ('objects+0', POLICIES[0]),
'': ('', POLICIES[0]),
'0': ('0', POLICIES[0]),
'-1': ('', POLICIES[1]),
}
for policy_string, expected in expectations.items():
if expected == PolicyError:
try:
invalid = split_policy_string(policy_string)
except PolicyError:
continue # good
else:
self.fail('The string %r returned %r '
'instead of raising a PolicyError' %
(policy_string, invalid))
self.assertEqual(expected, split_policy_string(policy_string))
# should be inverse of get_policy_string
self.assertEqual(policy_string, get_policy_string(*expected))
def test_split_policy_string(self):
expectations = {
"something": ("something", POLICIES[0]),
"something-1": ("something", POLICIES[1]),
"tmp": ("tmp", POLICIES[0]),
"objects": ("objects", POLICIES[0]),
"tmp-1": ("tmp", POLICIES[1]),
"objects-1": ("objects", POLICIES[1]),
"objects-": PolicyError,
"objects-0": PolicyError,
"objects--1": ("objects-", POLICIES[1]),
"objects-+1": PolicyError,
"objects--": PolicyError,
"objects-foo": PolicyError,
"objects--bar": PolicyError,
"objects-+bar": PolicyError,
# questionable, demonstrated as inverse of get_policy_string
"objects+0": ("objects+0", POLICIES[0]),
"": ("", POLICIES[0]),
"0": ("0", POLICIES[0]),
"-1": ("", POLICIES[1]),
}
for policy_string, expected in expectations.items():
if expected == PolicyError:
try:
invalid = split_policy_string(policy_string)
except PolicyError:
continue # good
else:
self.fail(
"The string %r returned %r " "instead of raising a PolicyError" % (policy_string, invalid)
)
self.assertEqual(expected, split_policy_string(policy_string))
# should be inverse of get_policy_string
self.assertEqual(policy_string, get_policy_string(*expected))
def check_on_disk_files(self, files):
for file_path in files:
# File's not there
self.assertPathDoesNotExist(file_path)
# Got quarantined
parts = file_path.split(os.path.sep)
policy_dir = get_policy_string('objects', self.policy)
quarantine_dir = parts[:parts.index(policy_dir)] + ['quarantined']
quarantine_path = os.path.sep.join(quarantine_dir + [policy_dir] +
parts[-2:])
self.assertPathExists(quarantine_path)
def gather_object_files_by_ext(self):
result = collections.defaultdict(set)
for node in self.brain.nodes:
for path, _, files in os.walk(
os.path.join(self.device_dir(node),
get_policy_string('objects', self.policy))):
for file in files:
if file in ('.lock', 'hashes.pkl', 'hashes.invalid'):
continue
_, ext = os.path.splitext(file)
result[ext].add(os.path.join(path, file))
return result
def iter_all_objects(self, conn, policy):
prefix = get_policy_string('objects', policy)
key_range = [prefix + term for term in ('.', '/')]
keys = conn.getKeyRange(*key_range).wait()
while keys:
for key in keys:
# TODO: clean up hashdir and old tombstones
yield key
# see if there's any more values
key_range[0] = key
keys = conn.getKeyRange(*key_range,
startKeyInclusive=False).wait()
def test_multiple_polices(self):
hash_ = swift_utils.hash_path('a', 'c', 'o')
t = swift_utils.Timestamp(time.time())
nonce = uuid.uuid4()
key_template = '%(prefix)s.%(hash)s.%(timestamp)s.%(ext)s' \
'.%(nonce)s%(trailing_frag)s'
with utils.patch_policies(with_ec_default=True):
for p in storage_policy.POLICIES:
if p.policy_type == storage_policy.EC_POLICY:
frag_index = random.randint(0, 10)
trailing_frag = '-%d' % (frag_index)
else:
frag_index = None
trailing_frag = ''
ext = random.choice(['data', 'ts'])
object_prefix = storage_policy.get_policy_string('objects', p)
key = key_template % {
'prefix': object_prefix,
'hash': hash_,
'timestamp': t.internal,
'ext': ext,
'nonce': nonce,
'trailing_frag': trailing_frag,
}
expected = {
'hashpath': hash_,
'policy': p,
'nonce': str(nonce),
'frag_index': frag_index,
'timestamp': t.internal,
'ext': ext,
}
try:
self.assertEqual(replicator.split_key(key), expected)
except AssertionError as e:
msg = '%s\n\n ... for key %r' % (e, key)
self.fail(msg)
def test_get_policy_string(self):
self.assertEquals(get_policy_string("something", 0), "something")
self.assertEquals(get_policy_string("something", None), "something")
self.assertEquals(get_policy_string("something", ""), "something")
self.assertEquals(get_policy_string("something", 1), "something" + "-1")
self.assertRaises(PolicyError, get_policy_string, "something", 99)