def outtestIndexes(self):
'''
now with some indexes...
this should contain
(3 choose 0) + (3 choose 1) + (3 choose 2) + (3 choose 3) = 8leaf nodes,
and 17 nodes in total
'''
LOG.info("\n\n === BBSearch Simple Test - 3 indexes === \n")
dc = designcandidates.DesignCandidates()
dc.addCollection("col1", [], [], [])
dc.addCollection("col2", [("key1",), ("key1", "key2"), ("key1", "key3")], [], [])
#same as above, just 4 time more leaf nodes, since c1 can be sharded on k1..3,None
bb = bbsearch.BBSearch(dc, self.costmodel, self.initialDesign, self.upper_bound, self.timeout)
bb.solve()
nodeList = bb.listAllNodes()
self.assertEqual(bb.totalNodes, len(nodeList))
self.assertEqual(bb.totalNodes, 17)
self.assertEqual(bb.leafNodes, 8)
self.assertTrue(checkIndexKeyExist(nodeList, ([])))
self.assertTrue(checkIndexKeyExist(nodeList, [("key1",)] ))
self.assertTrue(checkIndexKeyExist(nodeList, [("key1", "key2")] ))
self.assertTrue(checkIndexKeyExist(nodeList, [("key1", "key3")] ))
self.assertTrue(checkIndexKeyExist(nodeList, [("key1",), ("key1", "key2")] ))
self.assertTrue(checkIndexKeyExist(nodeList, [("key1",), ("key1", "key3")] ))
self.assertTrue(checkIndexKeyExist(nodeList, [("key1", "key2"), ("key1", "key3")] ))
self.assertTrue(checkIndexKeyExist(nodeList, [("key1",), ("key1", "key2"), ("key1", "key3")] ))
def outtestMoreShardingKeys1(self):
'''
this should contain
(3 choose 0) + (3 choose 1) + (3 choose 2) + (3 choose 3) = 8leaf nodes,
and 8*2 + 1 nodes in total
'''
LOG.info("\n\n === BBSearch Simple Test - more shard keys === \n")
dc = designcandidates.DesignCandidates()
dc.addCollection("col1", [], [], [])
dc.addCollection("col2", [], ["key1", "key2", "key3"], [])
#same as above, just 4 time more leaf nodes, since c1 can be sharded on k1..3,None
bb = bbsearch.BBSearch(dc, self.costmodel, self.initialDesign, self.upper_bound, self.timeout)
bb.solve()
nodeList = bb.listAllNodes()
#for n in nodeList:
# print n
self.assertEqual(bb.totalNodes, len(nodeList))
self.assertEqual(bb.totalNodes, 17)
self.assertEqual(bb.leafNodes, 8)
self.assertTrue(checkShardKeyExist(nodeList, ([])))
self.assertTrue(checkShardKeyExist(nodeList, ("key1",)))
self.assertTrue(checkShardKeyExist(nodeList, ("key2",)))
self.assertTrue(checkShardKeyExist(nodeList, ("key3",)))
self.assertTrue(checkShardKeyExist(nodeList, ("key1", "key2")))
self.assertTrue(checkShardKeyExist(nodeList, ("key1", "key3")))
self.assertTrue(checkShardKeyExist(nodeList, ("key2", "key3")))
self.assertTrue(checkShardKeyExist(nodeList, ("key1", "key2", "key3")))
def testBBSearch_timeoutTest():
print("\n\n === BBSerch Timeout Test === \n")
# cost function takes 1 sec... therefore, there should be about 10 nodes after 10 sec
def slow_bounding_f(design):
time.sleep(1)
return 0
initialDesign = design.Design()
upper_bound = 1
timeout = 5
costmodel = DummyCostModel(slow_bounding_f)
dc = designcandidate.DesignCandidate()
dc.addCollection("col1", [], ["key1", "key2", "key3"], [])
dc.addCollection("col2", [], ["field1", "field2"], [])
bb = bbsearch.BBSearch(dc, costmodel, initialDesign, upper_bound, timeout)
bb.solve()
nodeList = bb.listAllNodes()
def outtestMoreShardingKeys2(self):
'''
this should contain
(5 choose 0) + (5 choose 1) + (5 choose 2) + (5 choose 3) = 26 leaf nodes,
and 26*2 + 1 = 53 nodes in total
'''
LOG.info("\n\n === BBSearch Simple Test - even more shard keys === \n")
dc = designcandidates.DesignCandidates()
dc.addCollection("col1", [], [], [])
dc.addCollection("col2", [], ["key1", "key2", "key3", "key4", "key5"], [])
#same as above, just 4 time more leaf nodes, since c1 can be sharded on k1..3,None
bb = bbsearch.BBSearch(dc, self.costmodel, self.initialDesign, self.upper_bound, self.timeout)
bb.solve()
nodeList = bb.listAllNodes()
self.assertEqual(bb.totalNodes, len(nodeList))
self.assertEqual(bb.totalNodes, 53)
self.assertEqual(bb.leafNodes, 26)
def outtestSimpleSearch(self):
'''
dummy example: since the bounding function returns always float('inf'),
this should basically traverse the entire tree
--> this test verifies that bbsearch does not omit any nodes
'''
LOG.info("\n\n === BBSearch Simple Test - empty === \n")
dc = designcandidates.DesignCandidates()
dc.addCollection("col1", [], [], [])
dc.addCollection("col2", [], [], [])
LOG.info("Design Candidates\n%s", dc)
LOG.info("Initial Design\n%s", self.initialDesign)
bb = bbsearch.BBSearch(dc, self.costmodel, self.initialDesign, self.upper_bound, self.timeout)
bb.solve()
nodeList = bb.listAllNodes()
self.assertEqual(bb.totalNodes, len(nodeList))
self.assertEqual(bb.totalNodes, 3)
self.assertEqual(bb.leafNodes, 1)
def outtestDenormalization(self):
'''
now with some denorm...
3 leaf nodes: no denorm, col1->col2, col2->col1
'''
LOG.info("\n\n === BBSearch Simple Test - denorm === \n")
dc = designcandidates.DesignCandidates()
dc.addCollection("col1", [], [], ["col2"])
dc.addCollection("col2", [], [], ["col1"])
#same as above, just 4 time more leaf nodes, since c1 can be sharded on k1..3,None
bb = bbsearch.BBSearch(dc, self.costmodel, self.initialDesign, self.upper_bound, self.timeout)
bb.solve()
nodeList = bb.listAllNodes()
for n in nodeList:
print n
print "*"*50
self.assertEqual(bb.totalNodes, len(nodeList))
self.assertEqual(6, bb.totalNodes)
self.assertEqual(3, bb.leafNodes)
def outtestShardingKeys(self):
'''
now the some with shard keys...
this should contain 4 leaf nodes, 9 nodes in total
shard key ([]), ("key1"), ("key2"), ("key1", "key2")
'''
LOG.info("\n\n === BBSearch Simple Test - shard keys === \n")
dc = designcandidates.DesignCandidates()
dc.addCollection("col1", [], [], [])
dc.addCollection("col2", [], ["key1", "key2"], [])
#same as above, just 4 time more leaf nodes, since c1 can be sharded on k1..3,None
bb = bbsearch.BBSearch(dc, self.costmodel, self.initialDesign, self.upper_bound, self.timeout)
bb.solve()
nodeList = bb.listAllNodes()
#for n in nodeList:
# print n
self.assertEqual(bb.totalNodes, len(nodeList))
self.assertEqual(bb.totalNodes, 9)
self.assertEqual(bb.leafNodes, 4)
self.assertTrue(checkShardKeyExist(nodeList, ([])))
self.assertTrue(checkShardKeyExist(nodeList, ("key1",)))
self.assertTrue(checkShardKeyExist(nodeList, ("key2",)))
self.assertTrue(checkShardKeyExist(nodeList, ("key1", "key2")))
def run(self):
"""
main public method. Simply call to get the optimal solution
"""
col_generator = LNSDesigner.RandomCollectionGenerator(self.collections)
worker_used_time = 0 # This is used to record how long this worker has been running
elapsedTime = 0 # this is used to check if this worker has found a better design for a limited time: patient time
relaxRatio = self.init_relaxRatio
bbsearch_time_out = self.init_bbsearch_time
bestCost = self.init_bestCost
bestDesign = self.init_bestDesign.copy()
while True:
relaxedCollectionsNames, relaxedDesign = self.__relax__(
col_generator, bestDesign, relaxRatio)
sendMessage(
MSG_SEARCH_INFO,
(relaxedCollectionsNames, bbsearch_time_out, relaxedDesign,
worker_used_time, elapsedTime, self.worker_id), self.channel)
dc = self.designCandidates.getCandidates(relaxedCollectionsNames)
self.bbsearch_method = bbsearch.BBSearch(dc, self.costModel,
relaxedDesign, bestCost,
bbsearch_time_out,
self.channel,
self.bestLock)
self.bbsearch_method.solve()
worker_used_time += self.bbsearch_method.usedTime
if self.bbsearch_method.status != "updated_design":
if self.bbsearch_method.bestCost < bestCost:
bestCost = self.bbsearch_method.bestCost
bestDesign = self.bbsearch_method.bestDesign.copy()
elapsedTime = 0
else:
elapsedTime += self.bbsearch_method.usedTime
if self.isExhaustedSearch:
elapsedTime = INIFITY
if elapsedTime >= self.patient_time:
# if it haven't found a better design for one hour, give up
LOG.info(
"Haven't found a better design for %s minutes. QUIT",
elapsedTime)
break
relaxRatio += self.ratio_step
if relaxRatio > self.max_ratio:
relaxRatio = self.max_ratio
self.timeout -= self.bbsearch_method.usedTime
bbsearch_time_out += self.ratio_step / 0.1 * 30
if self.timeout <= 0:
break
## IF
else:
if self.bbsearch_method.bestCost < bestCost:
bestCost = self.bbsearch_method.bestCost
bestDesign = self.bbsearch_method.bestDesign.copy()
## IF
## ELSE
## WHILE
sendMessage(MSG_EXECUTE_COMPLETED, self.worker_id, self.channel)
def testBBSearch1():
def dummy_bounding_f(design):
return (0)
initialDesign = design.Design()
upper_bound = 1
timeout = 1000000000
costmodel = DummyCostModel(dummy_bounding_f)
'''
dummy example: since the bounding function returns always float('inf'),
this should basically traverse the entire tree
--> this test verifies that bbsearch does not omit any nodes
'''
print("\n\n === BBSearch Simple Test - empty === \n")
dc = designcandidate.DesignCandidate()
dc.addCollection("col1", [], [], [])
dc.addCollection("col2", [], [], [])
bb = bbsearch.BBSearch(dc, costmodel, initialDesign, upper_bound, timeout)
bb.solve()
nodeList = bb.listAllNodes()
'''
now the some with shard keys...
'''
print("\n\n === BBSearch Simple Test - shard keys === \n")
dc = designcandidate.DesignCandidate()
dc.addCollection("col1", [], [], [])
dc.addCollection("col2", [], ["key1", "key2"], [])
#same as above, just 4 time more leaf nodes, since c1 can be sharded on k1..3,None
bb = bbsearch.BBSearch(dc, costmodel, initialDesign, upper_bound, timeout)
bb.solve()
nodeList = bb.listAllNodes()
print("\n\n === BBSearch Simple Test - more shard keys === \n")
dc = designcandidate.DesignCandidate()
dc.addCollection("col1", [], ["f1", "f2"], [])
dc.addCollection("col2", [], ["key2", "key3"], [])
#same as above, just 4 time more leaf nodes, since c1 can be sharded on k1..3,None
bb = bbsearch.BBSearch(dc, costmodel, initialDesign, upper_bound, timeout)
bb.solve()
nodeList = bb.listAllNodes()
'''
now with some indexes...
'''
print("\n\n === BBSearch Simple Test - indexes === \n")
dc = designcandidate.DesignCandidate()
dc.addCollection("col1", [], [], [])
dc.addCollection("col2", [("key1"), ("key1", "key2"), ("key1", "key3")],
[], [])
#same as above, just 4 time more leaf nodes, since c1 can be sharded on k1..3,None
bb = bbsearch.BBSearch(dc, costmodel, initialDesign, upper_bound, timeout)
bb.solve()
nodeList = bb.listAllNodes()
'''
now with some denorm...
'''
print("\n\n === BBSearch Simple Test - denorm === \n")
dc = designcandidate.DesignCandidate()
dc.addCollection("col1", [], [], ["col2"])
dc.addCollection("col2", [], [], ["col1"])
#same as above, just 4 time more leaf nodes, since c1 can be sharded on k1..3,None
bb = bbsearch.BBSearch(dc, costmodel, initialDesign, upper_bound, timeout)
bb.solve()
nodeList = bb.listAllNodes()