def test_basic(self): store = TestingMemServerStore() brl_user = '******' store.create_user(User(brl_user)) brl_block = BRLBlock('user/user/block/master') publisher = TestPublisher(brl_user, store) publisher.publish( brl_block, { 'r1.h': ('r1', ['user/block/r2.h']), 'r2.h': ('r2', []), 'r3.h': ('r3', []) }) store = MemServerStore(store) translator = ReferenceTranslatorService(store, brl_user) ver = BlockVersion(brl_block, 0) missing = References() missing[ver].add('r1.h') closure = CompatibilityClosure(missing) full_graph = BlockVersionGraph() full_graph.add_nodes([ver]) build_compatibility_closure(translator, closure, {ver}, full_graph) self.assertEqual(References(), closure.broken) expected = set() expected.add('user/block/r1.h') expected.add('user/block/r2.h') self.assertEqual(expected, closure.block_cell_names)
def test_basic(self): store = TestingMemServerStore() brl_user = '******' store.create_user(User(brl_user)) brl_block = BRLBlock('user/user/block/master') publisher = TestPublisher(brl_user, store) publisher.publish(brl_block, {'r1.h': ('r1', ['user/block/r2.h']), 'r2.h': ('r2', []), 'r3.h': ('r3', [])}) store = MemServerStore(store) translator = ReferenceTranslatorService(store, brl_user) ver = BlockVersion(brl_block, 0) missing = References() missing[ver].add('r1.h') closure = CompatibilityClosure(missing) full_graph = BlockVersionGraph() full_graph.add_nodes([ver]) build_compatibility_closure(translator, closure, {ver}, full_graph) self.assertEqual(References(), closure.broken) expected = set() expected.add('user/block/r1.h') expected.add('user/block/r2.h') self.assertEqual(expected, closure.block_cell_names)
def test_find_two_files_at_a_time(self): """Starts with one include, finds it, then two more includes, finish when all are found. Found items are taken into account in FindRequest.existing """ NUM_FILES = 10 brl_a = BRLBlock('%s/%s/%s/master' % (self.user, self.user, 'blocka')) names_a = [BlockCellName(self.user + "/blocka/a%d.h" % i) for i in range(NUM_FILES)] resources_info = {"a%d.h" % i: ("a", []) for i in range(NUM_FILES)} publisher = TestPublisher(self.user, self.store) publisher.publish(brl_a, resources_info) # Use the same request object, to accumulate existing (already found) files request = FinderRequest() request.policy = Policy.default() version = BlockVersion(brl_a, 0) # The version is always the same for i in range(0, NUM_FILES, 2): declaration1 = CPPDeclaration(names_a[i]) declaration2 = CPPDeclaration(names_a[i + 1]) request.unresolved = {declaration1, declaration2} result = self.service.find(request, BiiResponse()) self.check_result(result, resolved=[(brl_a, 0, {names_a[i], names_a[i + 1]})]) # The found one is added to the existing, for next iteration request.existing[version][declaration1] = {names_a[i]} request.existing[version][declaration2] = {names_a[i + 1]} self.assertEqual(len(request.existing[version]), i + 2)
def test_find_two_files_at_a_time(self): """Starts with one include, finds it, then two more includes, finish when all are found. Found items are taken into account in FindRequest.existing """ NUM_FILES = 10 brl_a = BRLBlock('%s/%s/%s/master' % (self.user, self.user, 'blocka')) names_a = [ BlockCellName(self.user + "/blocka/a%d.h" % i) for i in range(NUM_FILES) ] resources_info = {"a%d.h" % i: ("a", []) for i in range(NUM_FILES)} publisher = TestPublisher(self.user, self.store) publisher.publish(brl_a, resources_info) # Use the same request object, to accumulate existing (already found) files request = FinderRequest() request.policy = Policy.default() version = BlockVersion(brl_a, 0) # The version is always the same for i in range(0, NUM_FILES, 2): declaration1 = CPPDeclaration(names_a[i]) declaration2 = CPPDeclaration(names_a[i + 1]) request.unresolved = {declaration1, declaration2} result = self.service.find(request, BiiResponse()) self.check_result(result, resolved=[(brl_a, 0, {names_a[i], names_a[i + 1]})]) # The found one is added to the existing, for next iteration request.existing[version][declaration1] = {names_a[i]} request.existing[version][declaration2] = {names_a[i + 1]} self.assertEqual(len(request.existing[version]), i + 2)
def setUp(self): """basic setup and creation of 10 versions with DEV-ALPHA-BETA-STABLE-DEV....""" BaseFinderTest.setUp(self) self.name = BlockCellName(self.user + "/blocka/resourcename") self.brl = BRLBlock('%s/%s/%s/master' % (self.user, self.user, 'blocka')) publisher = TestPublisher(self.user, self.store) for tag in [STABLE, BETA, ALPHA, DEV]: time.sleep(0.05) publisher.publish(self.brl, {'resourcename': ('a', [])}, tag)
def test_disallow_cycles(self): """ a find cannot return a block that is already in the src blocks of the hive """ brl_a = BRLBlock('%s/%s/%s/master' % (self.user, self.user, 'blocka')) name_a = BlockCellName(self.user + "/blocka/a.h") publisher = TestPublisher(self.user, self.store) publisher.publish(brl_a, {'a.h': ('a', [])}) request = self.build_unresolved_request(name_a) request.block_names.add(name_a.block_name) response = BiiResponse() result = self.service.find(request, response) # TODO: return a message to user saying that it is because a cycle self.assertIn('No block candidates found', str(response)) self.check_result(result, unresolved=request.unresolved)
def test_simple_find(self): '''Test including a single block with 1 file 1 version without further dependencies, always last version ''' brl_a = BRLBlock('%s/%s/%s/master' % (self.user, self.user, 'blocka')) name_a = BlockCellName(self.user + "/blocka/a.h") publisher = TestPublisher(self.user, self.store) for i in range(4): publisher.publish(brl_a, {'a.h': ('a', [])}) # self.publish(brl_a, name_a) request = self.build_unresolved_request(name_a) result = self.service.find(request, BiiResponse()) self.check_result(result, resolved=[(brl_a, i, {name_a})])
def test_files_from_different_blocks(self): """This test solves the bug that appeared when user included 2 different headers from different blocks A, B, such A->B, and the files included were not modified among different versions. The problem was the finder discarding subsequent hypothesis because it thought they will be the same based on ids (cells, contents) and dep_table. This is not true, they might not have changed, but their dependencies have changed, so every block_version must have it's own hypothesis """ brl_a = BRLBlock('%s/%s/%s/master' % (self.user, self.user, 'blocka')) names = [BlockCellName(self.user + "/blocka/path/to/a%d.h" % i) for i in range(1, 3)] resources_info = {"path/to/a%d.h" % i: ("a", [names[0]] if i == 2 else []) for i in range(1, 3)} publisher = TestPublisher(self.user, self.store) publisher.publish(brl_a, resources_info) version_a0 = BlockVersion(brl_a, 0) brl_b = BRLBlock('%s/%s/%s/master' % (self.user, self.user, 'blockb')) publisher.publish(brl_b, {'path/to/b.h': ('b', [names[1]])}, dep_versions=version_a0) # publish 2 versions more of B, with a new file b2 for i in range(2): publisher.publish(brl_b, {'path/to/b2.h': ('b', [])}, dep_versions=version_a0) #publish a new A version, changing file a1 publisher.publish(brl_a, {'path/to/a1.h': ('a', [])}) request = self.build_unresolved_request(['find_user/blocka/path/to/a2.h', 'find_user/blockb/path/to/b.h']) result = self.service.find(request, BiiResponse()) self.check_result(result, resolved=[(brl_a, 0, {'find_user/blocka/path/to/a2.h'}), (brl_b, 2, {'find_user/blockb/path/to/b.h'})])
def test_find_transitive(self): '''Test including a block with other dependencies, without conflicts ''' brl_a = BRLBlock('%s/%s/%s/master' % (self.user, self.user, 'blocka')) name_a = BlockCellName(self.user + "/blocka/a.h") publisher = TestPublisher(self.user, self.store) publisher.publish(brl_a, {'a.h': ('a', [])}) brl_b = BRLBlock('%s/%s/%s/master' % (self.user, self.user, 'blockb')) name_b = BlockCellName(self.user + "/blockb/b.h") #self.publish(brl_b, name_b, deps={name_b: name_a}, dep_versions=BlockVersion(brl_a, 0)) publisher.publish(brl_b, {'b.h': ('b', [name_a])}, dep_versions=BlockVersion(brl_a, 0)) request = self.build_unresolved_request(name_b) result = self.service.find(request, BiiResponse()) self.check_result(result, resolved=[(brl_b, 0, {name_b})])
def test_simple_update(self): '''Test finding updates in a simple block with 1 file with no more dependencies ''' brl_a = BRLBlock('%s/%s/%s/master' % (self.user, self.user, 'blocka')) name_a = BlockCellName(self.user + "/blocka/a.h") publisher = TestPublisher(self.user, self.store) publisher.publish(brl_a, {'a.h': ('a', [])}) #self.publish(brl_a, name_a) for i in range(1, 10): time.sleep(0.05) publisher.publish(brl_a, {'a.h': ('a', [])}) #self.publish(brl_a, name_a) request = self.build_update_request(BlockVersion(brl_a, i - 1), name_a) result = self.service.find(request, BiiResponse()) ref_updated = [(brl_a, i, {name_a})] self.check_result(result, updated=ref_updated)
def test_performance_breadth(self): store = MongoServerStore(self.conn, self.__class__.__name__) store.create_user(User("user2")) publisher = TestPublisher("user2", store) brl_block = BRLBlock('user2/user2/block/master') count = 1000 resource_info = {} for i in xrange(count): deps = DependencySet() if i > 0: deps = DependencySet() for j in range(max(0, i - 25), i): deps.explicit.add(BlockCellName('user2/block/cell%d.h' % j)) deps.resolved.add( CPPDeclaration('user2/block/cell%d.h' % j)) deps.unresolved.add(CPPDeclaration('path/to/file.h')) deps.implicit.add(BlockCellName('user2/block/cell%d.h' % j)) resource_info['cell%d.h' % i] = 'content %d' % i, deps publisher.publish(brl_block, resource_info) timer = self.get_timer() start_time = timer() store = MemServerStore(store) #print 'MEMSTORE SIZE 0', asizeof(store) / 1000000.0 translator = ReferenceTranslatorService(store, "user2") version = BlockVersion(brl_block, 0) missing = References() missing[version].add('cell%d.h' % (count - 1)) closure = CompatibilityClosure(missing) full_graph = BlockVersionGraph() full_graph.add_nodes([version]) build_compatibility_closure(translator, closure, [version], full_graph) elapsed_time = timer() - start_time #print 'Closure time', elapsed_time #print 'CLOSURE SIZE ', asizeof(closure) / 1000000.0 #print 'MEMSTORE SIZE ', asizeof(store) / 1000000.0 # print 'MINCELLS SIZE ', asizeof(store.min_cells)/1000000.0 self.assertEqual({brl_block.block_name + c for c in resource_info}, closure.block_cell_names) self.assertLess(elapsed_time, 7)
def test_performance_breadth(self): store = MongoServerStore(self.conn, self.__class__.__name__) store.create_user(User("user2")) publisher = TestPublisher("user2", store) brl_block = BRLBlock('user2/user2/block/master') count = 1000 resource_info = {} for i in xrange(count): deps = DependencySet() if i > 0: deps = DependencySet() for j in range(max(0, i - 25), i): deps.explicit.add(BlockCellName('user2/block/cell%d.h' % j)) deps.resolved.add(CPPDeclaration('user2/block/cell%d.h' % j)) deps.unresolved.add(CPPDeclaration('path/to/file.h')) deps.implicit.add(BlockCellName('user2/block/cell%d.h' % j)) resource_info['cell%d.h' % i] = 'content %d' % i, deps publisher.publish(brl_block, resource_info) timer = self.get_timer() start_time = timer() store = MemServerStore(store) #print 'MEMSTORE SIZE 0', asizeof(store) / 1000000.0 translator = ReferenceTranslatorService(store, "user2") version = BlockVersion(brl_block, 0) missing = References() missing[version].add('cell%d.h' % (count - 1)) closure = CompatibilityClosure(missing) full_graph = BlockVersionGraph() full_graph.add_nodes([version]) build_compatibility_closure(translator, closure, [version], full_graph) elapsed_time = timer() - start_time #print 'Closure time', elapsed_time #print 'CLOSURE SIZE ', asizeof(closure) / 1000000.0 #print 'MEMSTORE SIZE ', asizeof(store) / 1000000.0 # print 'MINCELLS SIZE ', asizeof(store.min_cells)/1000000.0 self.assertEqual({brl_block.block_name + c for c in resource_info}, closure.block_cell_names) self.assertLess(elapsed_time, 7)
def test_disallow_cycles_transitive(self): """ A find must not find any block name that is in the current src blocks, even transitively. """ brl_a = BRLBlock('%s/%s/%s/master' % (self.user, self.user, 'blocka')) name_a = BlockCellName(self.user + "/blocka/a.h") publisher = TestPublisher(self.user, self.store) publisher.publish(brl_a, {'a.h': ('a', [])}) brl_b = BRLBlock('%s/%s/%s/master' % (self.user, self.user, 'blockb')) name_b = BlockCellName(self.user + "/blockb/b.h") publisher.publish(brl_b, {'b.h': ('b', [name_a])}, dep_versions=BlockVersion(brl_a, 0)) request = self.build_unresolved_request(name_b) request.block_names.add(name_a.block_name) response = BiiResponse() result = self.service.find(request, response) self.assertIn("Can't find a compatible solution", str(response)) self.assertIn('it has cycles', str(response)) self.check_result(result, unresolved=request.unresolved)
def test_diamond_single_solution(self): brl_a = BRLBlock('%s/%s/%s/master' % (self.user, self.user, 'blocka')) brl_b = BRLBlock('%s/%s/%s/master' % (self.user, self.user, 'blockb')) brl_c = BRLBlock('%s/%s/%s/master' % (self.user, self.user, 'blockc')) name_a = BlockCellName(self.user + "/blocka/a.h") publisher = TestPublisher(self.user, self.store) for _ in range(20): publisher.publish(brl_a, {'a.h': ('a', [])}) name_b = BlockCellName(self.user + "/blockb/b.h") name_c = BlockCellName(self.user + "/blockc/c.h") for i in range(0, 20): if i % 2 == 0: publisher.publish(brl_b, {'b.h': ('b', [name_a])}, dep_versions=BlockVersion(brl_a, i)) if (i % 2 == 1 or i == 10): publisher.publish(brl_c, {'c.h': ('c', [name_a])}, dep_versions=BlockVersion(brl_a, i)) request = self.build_unresolved_request([name_b, name_c]) result = self.service.find(request, BiiResponse()) self.check_result(result, resolved=[(brl_b, 5, {name_b}), (brl_c, 5, {name_c})])
def test_performance_depth(self): brl_user = '******' store = MongoServerStore(self.conn, self.__class__.__name__) store.create_user(User(brl_user)) publisher = TestPublisher(brl_user, store) brl_block = BRLBlock('user/user/block/master') count = 500 resource_info = { 'cell%d.h' % i: ('content %d' % i, ['user/block/cell%d.h' % (i - 1)] if i else []) for i in range(count) } publisher.publish(brl_block, resource_info) timer = self.get_timer() start_time = timer() # prof = cProfile.Profile() # prof.enable() store = MemServerStore(store) # print 'MEMSTORE DEPTH SIZE 0', asizeof(store) / 1000000.0 translator = ReferenceTranslatorService(store, brl_user) version = BlockVersion(brl_block, 0) missing = References() missing[version].add('cell%d.h' % (count - 1)) closure = CompatibilityClosure(missing) full_graph = BlockVersionGraph() full_graph.add_nodes([version]) build_compatibility_closure(translator, closure, [version], full_graph) self.assertEqual(References(), closure.broken) #print 'CLOSURE SIZE ', asizeof(closure) / 1000000.0 #print 'MEMSTORE SIZE ', asizeof(store) / 1000000.0 elapsed_time = timer() - start_time #print 'Closure time', elapsed_time self.assertEqual({brl_block.block_name + c for c in resource_info}, closure.block_cell_names) self.assertLess(elapsed_time, 5)
def test_performance_depth(self): brl_user = '******' store = MongoServerStore(self.conn, self.__class__.__name__) store.create_user(User(brl_user)) publisher = TestPublisher(brl_user, store) brl_block = BRLBlock('user/user/block/master') count = 500 resource_info = {'cell%d.h' % i: ('content %d' % i, ['user/block/cell%d.h' % (i - 1)] if i else []) for i in range(count)} publisher.publish(brl_block, resource_info) timer = self.get_timer() start_time = timer() # prof = cProfile.Profile() # prof.enable() store = MemServerStore(store) # print 'MEMSTORE DEPTH SIZE 0', asizeof(store) / 1000000.0 translator = ReferenceTranslatorService(store, brl_user) version = BlockVersion(brl_block, 0) missing = References() missing[version].add('cell%d.h' % (count - 1)) closure = CompatibilityClosure(missing) full_graph = BlockVersionGraph() full_graph.add_nodes([version]) build_compatibility_closure(translator, closure, [version], full_graph) self.assertEqual(References(), closure.broken) #print 'CLOSURE SIZE ', asizeof(closure) / 1000000.0 #print 'MEMSTORE SIZE ', asizeof(store) / 1000000.0 elapsed_time = timer() - start_time #print 'Closure time', elapsed_time self.assertEqual({brl_block.block_name + c for c in resource_info}, closure.block_cell_names) self.assertLess(elapsed_time, 5)
def test_files_from_different_blocks(self): """This test solves the bug that appeared when user included 2 different headers from different blocks A, B, such A->B, and the files included were not modified among different versions. The problem was the finder discarding subsequent hypothesis because it thought they will be the same based on ids (cells, contents) and dep_table. This is not true, they might not have changed, but their dependencies have changed, so every block_version must have it's own hypothesis """ brl_a = BRLBlock('%s/%s/%s/master' % (self.user, self.user, 'blocka')) names = [ BlockCellName(self.user + "/blocka/path/to/a%d.h" % i) for i in range(1, 3) ] resources_info = { "path/to/a%d.h" % i: ("a", [names[0]] if i == 2 else []) for i in range(1, 3) } publisher = TestPublisher(self.user, self.store) publisher.publish(brl_a, resources_info) version_a0 = BlockVersion(brl_a, 0) brl_b = BRLBlock('%s/%s/%s/master' % (self.user, self.user, 'blockb')) publisher.publish(brl_b, {'path/to/b.h': ('b', [names[1]])}, dep_versions=version_a0) # publish 2 versions more of B, with a new file b2 for i in range(2): publisher.publish(brl_b, {'path/to/b2.h': ('b', [])}, dep_versions=version_a0) #publish a new A version, changing file a1 publisher.publish(brl_a, {'path/to/a1.h': ('a', [])}) request = self.build_unresolved_request( ['find_user/blocka/path/to/a2.h', 'find_user/blockb/path/to/b.h']) result = self.service.find(request, BiiResponse()) self.check_result(result, resolved=[ (brl_a, 0, {'find_user/blocka/path/to/a2.h'}), (brl_b, 2, {'find_user/blockb/path/to/b.h'}) ])