def testHashSpaceRepeatableLookups(self):
hashSpace = Flexihash()
for i in range(1, 10):
hashSpace.addTarget("target" + str(i))
self.assertEqual(hashSpace.lookup('t1'), hashSpace.lookup('t1'))
self.assertEqual(hashSpace.lookup('t2'), hashSpace.lookup('t2'))
def testAddTargetAndGetAllTargets(self):
hashSpace = Flexihash()
hashSpace \
.addTarget('t-a') \
.addTarget('t-b') \
.addTarget('t-c')
self.assertEqual(hashSpace.getAllTargets(), ['t-a', 't-b', 't-c'])
def testAddTargetAndGetAllTargets(self):
hashSpace = Flexihash()
hashSpace \
.addTarget('t-a') \
.addTarget('t-b') \
.addTarget('t-c')
self.assertEqual(hashSpace.getAllTargets(), ['t-a', 't-b', 't-c'])
def testHashSpaceLookupsAreValidTargets(self):
targets = ["target" + str(i) for i in range(1, 10)]
hashSpace = Flexihash()
hashSpace.addTargets(targets)
for i in range(1, 10):
self.assertTrue(hashSpace.lookup("r"+str(i)) in targets, 'target must be in list of targets')
def testGetMultipleTargetsWithOnlyOneTarget(self):
hashSpace = Flexihash()
hashSpace.addTarget("single-target")
targets = hashSpace.lookupList('resource', 2)
self.assertIsInstance(targets, list)
self.assertEqual(len(targets), 1)
self.assertEqual(targets[0], 'single-target')
def testGetMultipleTargetsWithOnlyOneTarget(self):
hashSpace = Flexihash()
hashSpace.addTarget("single-target")
targets = hashSpace.lookupList('resource', 2)
self.assertIsInstance(targets, list)
self.assertEqual(len(targets), 1)
self.assertEqual(targets[0], 'single-target')
def testGetMultipleTargets(self):
hashSpace = Flexihash()
for i in range(1, 10):
hashSpace.addTarget("target" + str(i))
targets = hashSpace.lookupList('resource', 2)
self.assertIsInstance(targets, list)
self.assertEqual(len(targets), 2)
self.assertNotEqual(targets[0], targets[1])
def testGetMoreTargetsThanExist(self):
hashSpace = Flexihash()
hashSpace.addTarget("target1")
hashSpace.addTarget("target2")
targets = hashSpace.lookupList('resource', 4)
self.assertIsInstance(targets, list)
self.assertEqual(len(targets), 2)
self.assertNotEqual(targets[0], targets[1])
def testGetMoreTargetsThanExist(self):
hashSpace = Flexihash()
hashSpace.addTarget("target1")
hashSpace.addTarget("target2")
targets = hashSpace.lookupList('resource', 4)
self.assertIsInstance(targets, list)
self.assertEqual(len(targets), 2)
self.assertNotEqual(targets[0], targets[1])
def testGetMultipleTargets(self):
hashSpace = Flexihash()
for i in range(1, 10):
hashSpace.addTarget("target" + str(i))
targets = hashSpace.lookupList('resource', 2)
self.assertIsInstance(targets, list)
self.assertEqual(len(targets), 2)
self.assertNotEqual(targets[0], targets[1])
def testHashSpaceLookupsAreValidTargets(self):
targets = ["target" + str(i) for i in range(1, 10)]
hashSpace = Flexihash()
hashSpace.addTargets(targets)
for i in range(1, 10):
self.assertTrue(
hashSpace.lookup("r" + str(i)) in targets,
'target must be in list of targets')
def testLookupListReturnsWithShortListIfAllTargetsUsed(self):
hashSpace = Flexihash()
# both have CRC32 of 1253617450
hashSpace.addTarget("x").addTarget("y") # make the list non-empty, non-one-value, to avoid shortcuts
hashSpace.addTarget("80726") # add a value
hashSpace.addTarget("14746907") # add a different value with the same hash, to clobber the first
hashSpace.removeTarget("14746907") # remove the fourth value; with the third clobbered, only X and Y are left
result = hashSpace.lookupList('test', 3) # try to get 3 results, our target list is X, Y, 80726
self.assertEqual(len(result), 2) # but 80726 isn't reachable since it was clobbered
self.assertIn("x", result) # all that's left is x
self.assertIn("y", result) # and y
def balance(tmpl):
"""
>>> balance("http://mirror-{001,002,003}.foo.com/filename01.png")
'http://mirror-002.foo.com/filename01.png'
>>> balance("http://mirror-{001,002,003}.foo.com/filename02.png")
'http://mirror-001.foo.com/filename02.png'
>>> balance("http://mirror-{001,002,003}.foo.com/filename03.png")
'http://mirror-002.foo.com/filename03.png'
>>> balance("http://mirror-{001,002,003}.foo.com/filename04.png")
'http://mirror-002.foo.com/filename04.png'
"""
matches = _balance_pattern.match(tmpl)
if matches:
pre = matches.group(1)
opts = matches.group(2)
post = matches.group(3)
if opts not in _balancers:
_balancers[opts] = Flexihash()
opt_list = opts.split(",")
for opt in opt_list:
if "=" in opt:
val, weight = opt.split("=")
else:
val, weight = opt, 1
_balancers[opts].addTarget(val, int(weight))
choice = _balancers[opts].lookup(
(pre + post).encode("ascii", "replace"))
return pre + choice + post
return tmpl
def test_basic(self):
fh = Flexihash()
fh.addTarget("a")
fh.addTarget("b")
fh.addTarget("c")
self.assertEqual(fh.lookup("1"), "a")
self.assertEqual(fh.lookup("2"), "b")
self.assertEqual(fh.lookup("3"), "a")
def testHashSpaceConsistentLookupsWithNewInstance(self):
hashSpace1 = Flexihash()
for i in range(1, 10):
hashSpace1.addTarget("target" + str(i))
results1 = []
for i in range(1, 100):
results1.append(hashSpace1.lookup("t" + str(i)))
hashSpace2 = Flexihash()
for i in range(1, 10):
hashSpace2.addTarget("target" + str(i))
results2 = []
for i in range(1, 100):
results2.append(hashSpace2.lookup("t" + str(i)))
self.assertEqual(results1, results2)
def testGetMultipleTargetsWithoutNeedingToLoopToStart(self):
mockHasher = MockHasher()
hashSpace = Flexihash(mockHasher, 1)
mockHasher.setHashValue(10)
hashSpace.addTarget("t1")
mockHasher.setHashValue(20)
hashSpace.addTarget("t2")
mockHasher.setHashValue(30)
hashSpace.addTarget("t3")
mockHasher.setHashValue(15)
targets = hashSpace.lookupList('resource', 2)
self.assertEqual(targets, ['t2', 't3'])
def testHashSpaceRepeatableLookups(self):
hashSpace = Flexihash()
for i in range(1, 10):
hashSpace.addTarget("target" + str(i))
self.assertEqual(hashSpace.lookup('t1'), hashSpace.lookup('t1'))
self.assertEqual(hashSpace.lookup('t2'), hashSpace.lookup('t2'))
def testHashSpaceConsistentLookupsAfterAddingAndRemoving(self):
hashSpace = Flexihash()
for i in range(1, 10):
hashSpace.addTarget("target" + str(i))
results1 = []
for i in range(1, 100):
results1.append(hashSpace.lookup("t"+str(i)))
hashSpace \
.addTarget('new-target') \
.removeTarget('new-target') \
.addTarget('new-target') \
.removeTarget('new-target')
results2 = []
for i in range(1, 100):
results2.append(hashSpace.lookup("t"+str(i)))
# This is probably optimistic, as adding/removing a target may
# clobber existing targets and is not expected to restore them.
self.assertEqual(results1, results2)
def testGetMultipleTargetsWithoutNeedingToLoopToStart(self):
mockHasher = MockHasher()
hashSpace = Flexihash(mockHasher, 1)
mockHasher.setHashValue(10)
hashSpace.addTarget("t1")
mockHasher.setHashValue(20)
hashSpace.addTarget("t2")
mockHasher.setHashValue(30)
hashSpace.addTarget("t3")
mockHasher.setHashValue(15)
targets = hashSpace.lookupList('resource', 2)
self.assertEqual(targets, ['t2', 't3'])
def testHashSpaceConsistentLookupsWithNewInstance(self):
hashSpace1 = Flexihash()
for i in range(1, 10):
hashSpace1.addTarget("target" + str(i))
results1 = []
for i in range(1, 100):
results1.append(hashSpace1.lookup("t"+str(i)))
hashSpace2 = Flexihash()
for i in range(1, 10):
hashSpace2.addTarget("target" + str(i))
results2 = []
for i in range(1, 100):
results2.append(hashSpace2.lookup("t"+str(i)))
self.assertEqual(results1, results2)
def testHashSpaceConsistentLookupsAfterAddingAndRemoving(self):
hashSpace = Flexihash()
for i in range(1, 10):
hashSpace.addTarget("target" + str(i))
results1 = []
for i in range(1, 100):
results1.append(hashSpace.lookup("t" + str(i)))
hashSpace \
.addTarget('new-target') \
.removeTarget('new-target') \
.addTarget('new-target') \
.removeTarget('new-target')
results2 = []
for i in range(1, 100):
results2.append(hashSpace.lookup("t" + str(i)))
# This is probably optimistic, as adding/removing a target may
# clobber existing targets and is not expected to restore them.
self.assertEqual(results1, results2)
def testLookupThrowsExceptionOnEmpty(self):
hashSpace = Flexihash()
self.assertRaises(Flexihash_Exception, hashSpace.lookup, 'test')
def testLookupListReturnsWithShortListIfAllTargetsUsed(self):
hashSpace = Flexihash()
# both have CRC32 of 1253617450
hashSpace.addTarget("x").addTarget(
"y") # make the list non-empty, non-one-value, to avoid shortcuts
hashSpace.addTarget("80726") # add a value
hashSpace.addTarget(
"14746907"
) # add a different value with the same hash, to clobber the first
hashSpace.removeTarget(
"14746907"
) # remove the fourth value; with the third clobbered, only X and Y are left
result = hashSpace.lookupList(
'test', 3) # try to get 3 results, our target list is X, Y, 80726
self.assertEqual(len(result),
2) # but 80726 isn't reachable since it was clobbered
self.assertIn("x", result) # all that's left is x
self.assertIn("y", result) # and y
def testLookupListThrowsExceptionOnZero(self):
hashSpace = Flexihash()
self.assertRaises(Flexihash_Exception, hashSpace.lookupList, 'test', 0)
def testGetMultipleTargetsNeedingToLoopToStart(self):
mockHasher = MockHasher()
hashSpace = Flexihash(mockHasher, 1)
mockHasher.setHashValue(10)
hashSpace.addTarget("t1")
mockHasher.setHashValue(20)
hashSpace.addTarget("t2")
mockHasher.setHashValue(30)
hashSpace.addTarget("t3")
mockHasher.setHashValue(40)
hashSpace.addTarget("t4")
mockHasher.setHashValue(50)
hashSpace.addTarget("t5")
mockHasher.setHashValue(35)
targets = hashSpace.lookupList('resource', 4)
self.assertEqual(targets, ['t4', 't5', 't1', 't2'])
def testAddTargetsAndGetAllTargets(self):
targets = ['t-a', 't-b', 't-c']
hashSpace = Flexihash()
hashSpace.addTargets(targets)
self.assertEqual(hashSpace.getAllTargets(), targets)
def testGetAllTargetsEmpty(self):
hashSpace = Flexihash()
self.assertEqual(hashSpace.getAllTargets(), [])
def testAddTargetThrowsExceptionOnDuplicateTarget(self):
hashSpace = Flexihash()
hashSpace.addTarget('t-a')
self.assertRaises(FlexihashException, hashSpace.addTarget, 't-a')
def testAddTargetsAndGetAllTargets(self):
targets = ['t-a', 't-b', 't-c']
hashSpace = Flexihash()
hashSpace.addTargets(targets)
self.assertEqual(hashSpace.getAllTargets(), targets)
def testFallbackPrecedenceWhenServerRemoved(self):
mockHasher = MockHasher()
hashSpace = Flexihash(mockHasher, 1)
mockHasher.setHashValue(10)
hashSpace.addTarget("t1")
mockHasher.setHashValue(20)
hashSpace.addTarget("t2")
mockHasher.setHashValue(30)
hashSpace.addTarget("t3")
mockHasher.setHashValue(15)
self.assertEqual(hashSpace.lookup('resource'), 't2')
self.assertEqual(
hashSpace.lookupList('resource', 3),
['t2', 't3', 't1']
)
hashSpace.removeTarget('t2')
self.assertEqual(hashSpace.lookup('resource'), 't3')
self.assertEqual(
hashSpace.lookupList('resource', 3),
['t3', 't1']
)
hashSpace.removeTarget('t3')
self.assertEqual(hashSpace.lookup('resource'), 't1')
self.assertEqual(
hashSpace.lookupList('resource', 3),
['t1']
)
def testGetMultipleTargetsNeedingToLoopToStart(self):
mockHasher = MockHasher()
hashSpace = Flexihash(mockHasher, 1)
mockHasher.setHashValue(10)
hashSpace.addTarget("t1")
mockHasher.setHashValue(20)
hashSpace.addTarget("t2")
mockHasher.setHashValue(30)
hashSpace.addTarget("t3")
mockHasher.setHashValue(40)
hashSpace.addTarget("t4")
mockHasher.setHashValue(50)
hashSpace.addTarget("t5")
mockHasher.setHashValue(35)
targets = hashSpace.lookupList('resource', 4)
self.assertEqual(targets, ['t4', 't5', 't1', 't2'])
def testRemoveTargetFailsOnMissingTarget(self):
hashSpace = Flexihash()
self.assertRaises(Flexihash_Exception, hashSpace.removeTarget,
'not-there')
def testFallbackPrecedenceWhenServerRemoved(self):
mockHasher = MockHasher()
hashSpace = Flexihash(mockHasher, 1)
mockHasher.setHashValue(10)
hashSpace.addTarget("t1")
mockHasher.setHashValue(20)
hashSpace.addTarget("t2")
mockHasher.setHashValue(30)
hashSpace.addTarget("t3")
mockHasher.setHashValue(15)
self.assertEqual(hashSpace.lookup('resource'), 't2')
self.assertEqual(hashSpace.lookupList('resource', 3),
['t2', 't3', 't1'])
hashSpace.removeTarget('t2')
self.assertEqual(hashSpace.lookup('resource'), 't3')
self.assertEqual(hashSpace.lookupList('resource', 3), ['t3', 't1'])
hashSpace.removeTarget('t3')
self.assertEqual(hashSpace.lookup('resource'), 't1')
self.assertEqual(hashSpace.lookupList('resource', 3), ['t1'])
def testGetAllTargetsEmpty(self):
hashSpace = Flexihash()
self.assertEqual(hashSpace.getAllTargets(), [])
def testAddTargetThrowsExceptionOnDuplicateTarget(self):
hashSpace = Flexihash()
hashSpace.addTarget('t-a')
self.assertRaises(Flexihash_Exception, hashSpace.addTarget, 't-a')
