def diamond_test(self, conflict):
""" Blocks: C
Deps: A, B, D1-D2
C -> A, B; A->D1, B->D2
"""
references = References()
references[va].add('a.h')
references[vb].add('b.h')
b = Resource(SimpleCell('user/blockb/b.h'))
a = Resource(SimpleCell('user/blocka/a.h'))
d1 = Resource(SimpleCell('user/blockd/d.h'))
if conflict:
d2 = Resource(SimpleCell('user/blockd/d.h', CPP))
else:
d2 = Resource(SimpleCell('user/blockd/d.h'))
a.cell.dependencies.explicit.add(d1.name)
b.cell.dependencies.explicit.add(d2.name)
tables = {va: [vd1], vb: [vd2], vd1: [], vd2: []}
api = FakeApi(zip([va, vb, vd1, vd2], [a, b, d1, d2]), tables)
base_table = BlockVersionTable([vc, va, vb])
biiout = OutputStream()
graph, closure, _ = build_closure(api, references, base_table, biiout=biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va, vb, vd1, vd2])
expected_graph.add_edge(va, vd1)
expected_graph.add_edge(vb, vd2)
self.assertEqual(expected_graph, graph)
self.assertEqual({a.name, b.name, d1.name}, set(closure.keys()))
if conflict:
self.assertIn('Incompatible dependency "user/blockd/d.h"', str(biiout))
else:
self.assertEqual("", str(biiout))
def test_basic(self):
store = TestingMemServerStore()
brl_user = '******'
store.create_user(User(brl_user))
brl_block = BRLBlock('user/user/block/master')
publisher = TestPublisher(brl_user, store)
publisher.publish(
brl_block, {
'r1.h': ('r1', ['user/block/r2.h']),
'r2.h': ('r2', []),
'r3.h': ('r3', [])
})
store = MemServerStore(store)
translator = ReferenceTranslatorService(store, brl_user)
ver = BlockVersion(brl_block, 0)
missing = References()
missing[ver].add('r1.h')
closure = CompatibilityClosure(missing)
full_graph = BlockVersionGraph()
full_graph.add_nodes([ver])
build_compatibility_closure(translator, closure, {ver}, full_graph)
self.assertEqual(References(), closure.broken)
expected = set()
expected.add('user/block/r1.h')
expected.add('user/block/r2.h')
self.assertEqual(expected, closure.block_cell_names)
def test_basic(self):
store = TestingMemServerStore()
brl_user = '******'
store.create_user(User(brl_user))
brl_block = BRLBlock('user/user/block/master')
publisher = TestPublisher(brl_user, store)
publisher.publish(brl_block, {'r1.h': ('r1', ['user/block/r2.h']),
'r2.h': ('r2', []),
'r3.h': ('r3', [])})
store = MemServerStore(store)
translator = ReferenceTranslatorService(store, brl_user)
ver = BlockVersion(brl_block, 0)
missing = References()
missing[ver].add('r1.h')
closure = CompatibilityClosure(missing)
full_graph = BlockVersionGraph()
full_graph.add_nodes([ver])
build_compatibility_closure(translator, closure, {ver}, full_graph)
self.assertEqual(References(), closure.broken)
expected = set()
expected.add('user/block/r1.h')
expected.add('user/block/r2.h')
self.assertEqual(expected, closure.block_cell_names)
def overwrite_test(self):
""" Blocks: C (defines in requirements B2)
Deps: A, B2
C -> A -> B1
"""
references = References()
references[va].add('a.h')
b = Resource(SimpleCell('user/blockb/b.h'))
b2 = Resource(SimpleCell('user/blockb/b.h'), CPP)
a = Resource(SimpleCell('user/blocka/a.h'))
a2 = Resource(SimpleCell('user/blocka/a2.h'))
a.cell.dependencies.explicit.add(a2.name)
a2.cell.dependencies.explicit.add(b.name)
tables = {va: [vb], vb: [], vb2: []}
api = FakeApi(zip([va, va, vb, vb2], [a, a2, b, b2]), tables)
base_table = BlockVersionTable([vc, va, vb2]) # Note B2 defined here
biiout = OutputStream()
graph, closure, _ = build_closure(api, references, base_table, biiout=biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va, vb2])
expected_graph.add_edge(va, vb2)
self.assertEqual(expected_graph, graph)
expected_closure = Closure({a.name: ClosureItem(a, va),
a2.name: ClosureItem(a2, va),
b.name: ClosureItem(b2, vb2)})
self.assertEqual(expected_closure, closure)
self.assertEqual("", str(biiout))
def simple_test(self):
""" Blocks: C
Deps: A, B
C -> A -> B
"""
references = References()
references[va].add('a.h')
base_table = BlockVersionTable([vc, va]) # The result including or excluding va is same
b = Resource(SimpleCell('user/blockb/b.h'))
a = Resource(SimpleCell('user/blocka/a.h'))
a2 = Resource(SimpleCell('user/blocka/a2.h'))
a.cell.dependencies.explicit.add(a2.name)
a2.cell.dependencies.explicit.add(b.name)
tables = {va: [vb], vb: []}
api = FakeApi(zip([va, va, vb], [a, a2, b]), tables)
biiout = OutputStream()
graph, closure, _ = build_closure(api, references, base_table, biiout=biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va, vb])
expected_graph.add_edge(va, vb)
self.assertEqual(expected_graph, graph)
expected_closure = Closure({a.name: ClosureItem(a, va),
a2.name: ClosureItem(a2, va),
b.name: ClosureItem(b, vb)})
self.assertEqual(expected_closure, closure)
self.assertEqual("", str(biiout))
def __init__(self):
self.dep_table = None # BlockVersionTable()
self.references = None # References()
# TODO: The whole closure is being serialized. Check performance, improve
self.closure = Closure() # {BlockCellName: Resource}
self.src_graph = BlockVersionGraph() # BlockVersionGraph
self.dep_graph = BlockVersionGraph() # BlockVersionGraph
def dep_overwriten_in_blocks_test(self):
""" Blocks: C, B (None version)
Deps: A
C -> A -> B
"""
references = References()
references[va].add('a.h')
vbn = BlockVersion(vb.block, None)
b = Resource(SimpleCell('user/blockb/b.h'))
a = Resource(SimpleCell('user/blocka/a.h'))
a2 = Resource(SimpleCell('user/blocka/a2.h'))
a.cell.dependencies.explicit.add(a2.name)
a2.cell.dependencies.explicit.add(b.name)
tables = {va: [vb], vb: []}
api = FakeApi(zip([va, va, vb], [a, a2, b]), tables)
base_table = BlockVersionTable([vc, vbn])
biiout = OutputStream()
graph, closure, _ = build_closure(api, references, base_table, biiout=biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va])
expected_graph.add_edge(va, vbn)
self.assertEqual(expected_graph, graph)
expected_closure = Closure({a.name: ClosureItem(a, va),
a2.name: ClosureItem(a2, va)})
self.assertEqual(expected_closure, closure)
self.assertEqual("", str(biiout))
def test_one_level(self):
base_versions = [a1]
api = DepsApiFake({a1: [b1],
b1: []})
graph, overwrites = block_version_graph_build(api.get_dep_table, base_versions, {})
expected = BlockVersionGraph()
expected.add_nodes([a1, b1])
expected.add_edge(a1, b1)
self.assertEqual(graph, expected)
def deserialize(data):
try:
res = HiveDependencies()
res.dep_table = BlockVersionTable.deserialize(data[HiveDependencies.SERIAL_DEP_TABLE])
res.references = References.deserialize(data[HiveDependencies.SERIAL_REFERENCES])
res.closure = Closure.deserialize(data[HiveDependencies.SERIAL_CLOSURE])
res.src_graph = BlockVersionGraph.deserialize(data[HiveDependencies.SERIAL_SRC_GRAPH])
res.dep_graph = BlockVersionGraph.deserialize(data[HiveDependencies.SERIAL_DEP_GRAPH])
return res
except Exception as e:
raise BiiSerializationException(e)
def test_same_tables(self):
base_versions = [c1, d1]
api = DepsApiFake({c1: [a2],
d1: [b2]})
graph, overwrites = block_version_graph_build(api.get_dep_table, base_versions,
BlockVersionTable([an, bn, c1, d1]))
expected = BlockVersionGraph()
expected.add_nodes([c1, d1])
expected.add_edge(c1, an)
expected.add_edge(d1, bn)
self.assertEqual(graph, expected)
self.assertEqual({c1: {an}, d1: {bn}}, overwrites)
def test_diamond_2_version(self):
base_versions = [a1, b1]
api = DepsApiFake({a1: [c1],
b1: [c2],
c1: [], c2: []})
graph, overwrites = block_version_graph_build(api.get_dep_table, base_versions, {})
expected = BlockVersionGraph()
expected.add_nodes([a1, b1, c1, c2])
expected.add_edge(a1, c1)
expected.add_edge(b1, c2)
self.assertEqual(graph, expected)
self.assertEqual({}, overwrites)
def test_effective_overwrite_none(self):
base_versions = [a1, b1]
api = DepsApiFake({a1: [c1],
b1: [c2],
cn: []})
graph, overwrites = block_version_graph_build(api.get_dep_table, base_versions,
BlockVersionTable([cn]))
expected = BlockVersionGraph()
expected.add_nodes([a1, b1, cn])
expected.add_edge(a1, cn)
expected.add_edge(b1, cn)
self.assertEqual(graph, expected)
self.assertEqual({a1: {cn}, b1: {cn}}, overwrites)
def disjoints_graphs_no_collisions_test(self):
g1 = BlockVersionGraph()
g2 = BlockVersionGraph()
self.assertEqual(BlockVersionGraph(), g1.collision(g2))
self.assertEqual(BlockVersionGraph(), g2.collision(g1))
brl0 = BRLBlock("user/user/block/master")
brl1 = BRLBlock("user/user/block2/master")
v0 = BlockVersion(brl0, 0)
v1 = BlockVersion(brl1, 1)
g1.add_node(v0)
g2.add_node(v1)
self.assertEqual(BlockVersionGraph(), g1.collision(g2))
self.assertEqual(BlockVersionGraph(), g2.collision(g1))
def block_version_graph_build(get_dep_table, base_versions, base_table):
graph = BlockVersionGraph()
overwrites = defaultdict(set)
for version in base_versions:
_recursive_block_version_graph_build(get_dep_table, graph, version,
base_table, overwrites)
return graph, overwrites
def deserialize(data):
try:
res = HiveDependencies()
res.dep_table = BlockVersionTable.deserialize(
data[HiveDependencies.SERIAL_DEP_TABLE])
res.references = References.deserialize(
data[HiveDependencies.SERIAL_REFERENCES])
res.closure = Closure.deserialize(
data[HiveDependencies.SERIAL_CLOSURE])
res.src_graph = BlockVersionGraph.deserialize(
data[HiveDependencies.SERIAL_SRC_GRAPH])
res.dep_graph = BlockVersionGraph.deserialize(
data[HiveDependencies.SERIAL_DEP_GRAPH])
return res
except Exception as e:
raise BiiSerializationException(e)
def test_performance_breadth(self):
store = MongoServerStore(self.conn, self.__class__.__name__)
store.create_user(User("user2"))
publisher = TestPublisher("user2", store)
brl_block = BRLBlock('user2/user2/block/master')
count = 1000
resource_info = {}
for i in xrange(count):
deps = DependencySet()
if i > 0:
deps = DependencySet()
for j in range(max(0, i - 25), i):
deps.explicit.add(BlockCellName('user2/block/cell%d.h' %
j))
deps.resolved.add(
CPPDeclaration('user2/block/cell%d.h' % j))
deps.unresolved.add(CPPDeclaration('path/to/file.h'))
deps.implicit.add(BlockCellName('user2/block/cell%d.h' % j))
resource_info['cell%d.h' % i] = 'content %d' % i, deps
publisher.publish(brl_block, resource_info)
timer = self.get_timer()
start_time = timer()
store = MemServerStore(store)
#print 'MEMSTORE SIZE 0', asizeof(store) / 1000000.0
translator = ReferenceTranslatorService(store, "user2")
version = BlockVersion(brl_block, 0)
missing = References()
missing[version].add('cell%d.h' % (count - 1))
closure = CompatibilityClosure(missing)
full_graph = BlockVersionGraph()
full_graph.add_nodes([version])
build_compatibility_closure(translator, closure, [version], full_graph)
elapsed_time = timer() - start_time
#print 'Closure time', elapsed_time
#print 'CLOSURE SIZE ', asizeof(closure) / 1000000.0
#print 'MEMSTORE SIZE ', asizeof(store) / 1000000.0
# print 'MINCELLS SIZE ', asizeof(store.min_cells)/1000000.0
self.assertEqual({brl_block.block_name + c
for c in resource_info}, closure.block_cell_names)
self.assertLess(elapsed_time, 7)
def virtual_test(self):
references = References()
references[va].add('a.h')
code = ('def virtual(settings):\n\tif(settings.os.family == "windows"):return "win"\n'
'\telse: return "nix"')
a = Resource(VirtualCell('user/blocka/a.h', code, {'win', 'nix'}))
awin = Resource(SimpleCell('user/blocka/win/a.h'))
anix = Resource(SimpleCell('user/blocka/nix/a.h'))
b = Resource(SimpleCell('user/blockb/b.h'))
d1 = Resource(SimpleCell('user/blockd/d.h'))
awin.cell.dependencies.explicit.add(b.name)
anix.cell.dependencies.explicit.add(d1.name)
tables = {va: [vb, vd1], vb: [], vd1: []}
api = FakeApi(zip([va, va, va, vb, vd1], [a, awin, anix, b, d1]), tables)
#With windows settings
settings = Settings(OSInfo(OSFamily("Windows")))
biiout = OutputStream()
graph, closure, _ = build_closure(api, references, {}, settings, biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va, vb])
expected_graph.add_edge(va, vb)
self.assertEqual(expected_graph, graph)
expected_closure = Closure({a.name: ClosureItem(a, va),
awin.name: ClosureItem(awin, va),
b.name: ClosureItem(b, vb),
})
self.assertEqual(expected_closure, closure)
self.assertEqual("", str(biiout))
#Change settings
settings = Settings(OSInfo(OSFamily("Linux")))
biiout = OutputStream()
graph, closure, _ = build_closure(api, references, {}, settings, biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va, vd1])
expected_graph.add_edge(va, vd1)
self.assertEqual(expected_graph, graph)
expected_closure = Closure({a.name: ClosureItem(a, va),
anix.name: ClosureItem(anix, va),
d1.name: ClosureItem(d1, vd1),
})
self.assertEqual(expected_closure, closure)
self.assertEqual("", str(biiout))
def test_performance_breadth(self):
store = MongoServerStore(self.conn, self.__class__.__name__)
store.create_user(User("user2"))
publisher = TestPublisher("user2", store)
brl_block = BRLBlock('user2/user2/block/master')
count = 1000
resource_info = {}
for i in xrange(count):
deps = DependencySet()
if i > 0:
deps = DependencySet()
for j in range(max(0, i - 25), i):
deps.explicit.add(BlockCellName('user2/block/cell%d.h' % j))
deps.resolved.add(CPPDeclaration('user2/block/cell%d.h' % j))
deps.unresolved.add(CPPDeclaration('path/to/file.h'))
deps.implicit.add(BlockCellName('user2/block/cell%d.h' % j))
resource_info['cell%d.h' % i] = 'content %d' % i, deps
publisher.publish(brl_block, resource_info)
timer = self.get_timer()
start_time = timer()
store = MemServerStore(store)
#print 'MEMSTORE SIZE 0', asizeof(store) / 1000000.0
translator = ReferenceTranslatorService(store, "user2")
version = BlockVersion(brl_block, 0)
missing = References()
missing[version].add('cell%d.h' % (count - 1))
closure = CompatibilityClosure(missing)
full_graph = BlockVersionGraph()
full_graph.add_nodes([version])
build_compatibility_closure(translator, closure, [version], full_graph)
elapsed_time = timer() - start_time
#print 'Closure time', elapsed_time
#print 'CLOSURE SIZE ', asizeof(closure) / 1000000.0
#print 'MEMSTORE SIZE ', asizeof(store) / 1000000.0
# print 'MINCELLS SIZE ', asizeof(store.min_cells)/1000000.0
self.assertEqual({brl_block.block_name + c for c in resource_info},
closure.block_cell_names)
self.assertLess(elapsed_time, 7)
def compute_src_graph(hive_holder, common_table):
""" computes just the src part of the full version graph.
Side effect: updates requirements of blocks to actually point to real dep versions
"""
graph = BlockVersionGraph()
versions = hive_holder.versions
graph.add_nodes(versions.itervalues())
references = References()
for block_holder in hive_holder.block_holders:
dep_table = block_holder.requirements
base_version = versions[block_holder.block_name]
for target_bcn in block_holder.external_targets():
target_block_name = target_bcn.block_name
if target_block_name in versions:
other_version = versions[target_block_name]
else:
other_version = common_table[target_block_name]
references[other_version].add(target_bcn.cell_name)
graph.add_edge(base_version, other_version)
dep_table.add_version(other_version)
return graph, references
def dep_overwriten_in_blocks_test(self):
""" Blocks: C, B (None version)
Deps: A
C -> A -> B
"""
references = References()
references[va].add('a.h')
vbn = BlockVersion(vb.block, None)
b = Resource(SimpleCell('user/blockb/b.h'))
a = Resource(SimpleCell('user/blocka/a.h'))
a2 = Resource(SimpleCell('user/blocka/a2.h'))
a.cell.dependencies.explicit.add(a2.name)
a2.cell.dependencies.explicit.add(b.name)
tables = {va: [vb], vb: []}
api = FakeApi(zip([va, va, vb], [a, a2, b]), tables)
base_table = BlockVersionTable([vc, vbn])
biiout = OutputStream()
graph, closure, _ = build_closure(api,
references,
base_table,
biiout=biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va])
expected_graph.add_edge(va, vbn)
self.assertEqual(expected_graph, graph)
expected_closure = Closure({
a.name: ClosureItem(a, va),
a2.name: ClosureItem(a2, va)
})
self.assertEqual(expected_closure, closure)
self.assertEqual("", str(biiout))
def _virtual_setup(self):
ref_translator = Mock()
depA = BlockVersion(BRLBlock('user/user/blockA/branch'), 4)
depB = BlockVersion(BRLBlock('user/user/blockB/branch'), 2)
baseC = BlockVersion(BRLBlock('user/user/blockC/branch'), 3)
full_graph = BlockVersionGraph()
full_graph.add_nodes([baseC, depA, depB])
full_graph.add_edge(baseC, depA)
full_graph.add_edge(baseC, depB)
def res_method(*args):
c_virtual_leaves = [BlockCellName('user/blockC/win/c.h'),
BlockCellName('user/blockC/nix/c.h')]
c_win_deps = [BlockCellName('user/blockA/a.h')]
c_nix_deps = [BlockCellName('user/blockB/b.h')]
result = ReferencedResources()
for ref in args[0].explode():
result[ref.block_version][ref.ref] = \
{Reference(depA, 'a.h'): ((0, 0), 0, []),
Reference(depB, 'b.h'): ((1, 1), 1, []),
Reference(baseC, 'c.h'): ((2, 2), 2, c_virtual_leaves),
Reference(baseC, 'win/c.h'): ((3, 3), 3, c_win_deps),
Reference(baseC, 'nix/c.h'): ((4, 4), 4, c_nix_deps)}[ref]
return result
ref_translator.get_published_min_refs.side_effect = res_method
return ref_translator, depA, depB, baseC, full_graph
def overwrite_test(self):
""" Blocks: C (defines in requirements B2)
Deps: A, B2
C -> A -> B1
"""
references = References()
references[va].add('a.h')
b = Resource(SimpleCell('user/blockb/b.h'))
b2 = Resource(SimpleCell('user/blockb/b.h'), CPP)
a = Resource(SimpleCell('user/blocka/a.h'))
a2 = Resource(SimpleCell('user/blocka/a2.h'))
a.cell.dependencies.explicit.add(a2.name)
a2.cell.dependencies.explicit.add(b.name)
tables = {va: [vb], vb: [], vb2: []}
api = FakeApi(zip([va, va, vb, vb2], [a, a2, b, b2]), tables)
base_table = BlockVersionTable([vc, va, vb2]) # Note B2 defined here
biiout = OutputStream()
graph, closure, _ = build_closure(api,
references,
base_table,
biiout=biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va, vb2])
expected_graph.add_edge(va, vb2)
self.assertEqual(expected_graph, graph)
expected_closure = Closure({
a.name: ClosureItem(a, va),
a2.name: ClosureItem(a2, va),
b.name: ClosureItem(b2, vb2)
})
self.assertEqual(expected_closure, closure)
self.assertEqual("", str(biiout))
def simple_test(self):
""" Blocks: C
Deps: A, B
C -> A -> B
"""
references = References()
references[va].add('a.h')
base_table = BlockVersionTable(
[vc, va]) # The result including or excluding va is same
b = Resource(SimpleCell('user/blockb/b.h'))
a = Resource(SimpleCell('user/blocka/a.h'))
a2 = Resource(SimpleCell('user/blocka/a2.h'))
a.cell.dependencies.explicit.add(a2.name)
a2.cell.dependencies.explicit.add(b.name)
tables = {va: [vb], vb: []}
api = FakeApi(zip([va, va, vb], [a, a2, b]), tables)
biiout = OutputStream()
graph, closure, _ = build_closure(api,
references,
base_table,
biiout=biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va, vb])
expected_graph.add_edge(va, vb)
self.assertEqual(expected_graph, graph)
expected_closure = Closure({
a.name: ClosureItem(a, va),
a2.name: ClosureItem(a2, va),
b.name: ClosureItem(b, vb)
})
self.assertEqual(expected_closure, closure)
self.assertEqual("", str(biiout))
def test_performance_depth(self):
brl_user = '******'
store = MongoServerStore(self.conn, self.__class__.__name__)
store.create_user(User(brl_user))
publisher = TestPublisher(brl_user, store)
brl_block = BRLBlock('user/user/block/master')
count = 500
resource_info = {
'cell%d.h' % i:
('content %d' % i, ['user/block/cell%d.h' % (i - 1)] if i else [])
for i in range(count)
}
publisher.publish(brl_block, resource_info)
timer = self.get_timer()
start_time = timer()
# prof = cProfile.Profile()
# prof.enable()
store = MemServerStore(store)
# print 'MEMSTORE DEPTH SIZE 0', asizeof(store) / 1000000.0
translator = ReferenceTranslatorService(store, brl_user)
version = BlockVersion(brl_block, 0)
missing = References()
missing[version].add('cell%d.h' % (count - 1))
closure = CompatibilityClosure(missing)
full_graph = BlockVersionGraph()
full_graph.add_nodes([version])
build_compatibility_closure(translator, closure, [version], full_graph)
self.assertEqual(References(), closure.broken)
#print 'CLOSURE SIZE ', asizeof(closure) / 1000000.0
#print 'MEMSTORE SIZE ', asizeof(store) / 1000000.0
elapsed_time = timer() - start_time
#print 'Closure time', elapsed_time
self.assertEqual({brl_block.block_name + c
for c in resource_info}, closure.block_cell_names)
self.assertLess(elapsed_time, 5)
def test_performance_depth(self):
brl_user = '******'
store = MongoServerStore(self.conn, self.__class__.__name__)
store.create_user(User(brl_user))
publisher = TestPublisher(brl_user, store)
brl_block = BRLBlock('user/user/block/master')
count = 500
resource_info = {'cell%d.h' % i: ('content %d' % i,
['user/block/cell%d.h' % (i - 1)] if i else [])
for i in range(count)}
publisher.publish(brl_block, resource_info)
timer = self.get_timer()
start_time = timer()
# prof = cProfile.Profile()
# prof.enable()
store = MemServerStore(store)
# print 'MEMSTORE DEPTH SIZE 0', asizeof(store) / 1000000.0
translator = ReferenceTranslatorService(store, brl_user)
version = BlockVersion(brl_block, 0)
missing = References()
missing[version].add('cell%d.h' % (count - 1))
closure = CompatibilityClosure(missing)
full_graph = BlockVersionGraph()
full_graph.add_nodes([version])
build_compatibility_closure(translator, closure, [version], full_graph)
self.assertEqual(References(), closure.broken)
#print 'CLOSURE SIZE ', asizeof(closure) / 1000000.0
#print 'MEMSTORE SIZE ', asizeof(store) / 1000000.0
elapsed_time = timer() - start_time
#print 'Closure time', elapsed_time
self.assertEqual({brl_block.block_name + c for c in resource_info},
closure.block_cell_names)
self.assertLess(elapsed_time, 5)
def test_build_closure_different_versions_restricted(self):
'''Computes a CompatibilityClosure in which two versions of blockA actually point to the
same unmodified resource with the typical diamond layout
Also computes and checks the BlockClosure for that layout'''
ref_translator = Mock()
depA1 = BlockVersion(BRLBlock('user/user/blockA/branch'), 4)
depA2 = BlockVersion(BRLBlock('user/user/blockA/branch'), 5)
baseB = BlockVersion(BRLBlock('user/user/blockB/branch'), 2)
baseC = BlockVersion(BRLBlock('user/user/blockC/branch'), 3)
full_graph = BlockVersionGraph()
full_graph.add_nodes([baseB, baseC, depA1, depA2])
full_graph.add_edge(baseB, depA1)
full_graph.add_edge(baseC, depA2)
def res_method(*args):
depsb = [BlockCellName('user/blockA/a.h')]
depsc = [BlockCellName('user/blockA/a.h')]
result = ReferencedResources()
for ref in args[0].explode():
result[ref.block_version][ref.ref] = \
{Reference(depA1, 'a.h'): ((0, 0), 0, []),
Reference(depA2, 'a.h'): ((0, 0), 0, []),
Reference(baseB, 'b.h'): ((1, 4), 1, depsb),
Reference(baseC, 'c.h'): ((2, 3), 2, depsc)}[ref]
return result
ref_translator.get_published_min_refs.side_effect = res_method
missing = References()
missing.add(Reference(baseB, 'b.h'))
missing.add(Reference(baseC, 'c.h'))
closure = CompatibilityClosure(missing)
build_compatibility_closure(ref_translator, closure, {baseC}, full_graph)
self.assertEqual(References(), closure.broken)
self.assertEqual({BlockCellName('user/blockC/c.h')}, closure.block_cell_names)
#self.assertIn(Reference(depA1, 'a.h'), closure.references)
self.assertNotIn(baseB, closure.references)
self.assertNotIn(depA1, closure.references)
self.assertNotIn(depA2, closure.references)
self.assertIn(baseC, closure.references)
expected_frontier = References()
expected_frontier[baseB].add('b.h')
expected_frontier[depA2].add('a.h')
self.assertEqual(expected_frontier, closure.frontier)
def compute_src_graph(hive_holder, common_table):
""" computes just the src part of the full version graph.
Side effect: updates requirements of blocks to actually point to real dep versions
"""
graph = BlockVersionGraph()
versions = hive_holder.versions
graph.add_nodes(versions.itervalues())
references = References()
for block_holder in hive_holder.block_holders:
dep_table = block_holder.requirements
base_version = versions[block_holder.block_name]
for target_bcn in block_holder.external_targets():
target_block_name = target_bcn.block_name
if target_block_name in versions:
other_version = versions[target_block_name]
else:
other_version = common_table[target_block_name]
references[other_version].add(target_bcn.cell_name)
graph.add_edge(base_version, other_version)
dep_table.add_version(other_version)
return graph, references
def test_broken_closure(self):
'''computes a closure in which cell blockB/b.h depends on blockA/a.h, but that a.h
is not found. The algorithms of closures must return that as a missing or broken dependency
'''
ref_translator = Mock()
baseB = BlockVersion(BRLBlock('user/user/blockB/branch'), 2)
depA1 = BlockVersion(BRLBlock('user/user/blockA/branch'), 4)
full_graph = BlockVersionGraph()
full_graph.add_nodes([baseB, depA1])
full_graph.add_edge(baseB, depA1)
def res_method(*args):
depsb = [BlockCellName('user/blockA/a.h')]
result = ReferencedResources()
for ref in args[0].explode():
try:
result[ref.block_version][ref.ref] = \
{
Reference(baseB, 'b.h'): ((0, 0), 0, depsb),
}[ref]
except KeyError:
pass
return result
ref_translator.get_published_min_refs.side_effect = res_method
missing = References()
missing.add(Reference(baseB, 'b.h'))
closure = CompatibilityClosure(missing)
build_compatibility_closure(ref_translator, closure, full_graph.nodes, full_graph)
self.assertEqual({BlockCellName('user/blockB/b.h')}, closure.block_cell_names)
self.assertIn(baseB, closure.references)
expected_missing = References()
expected_missing[depA1].add('a.h')
self.assertEqual(closure.broken, expected_missing)
def diamond_test(self, conflict):
""" Blocks: C
Deps: A, B, D1-D2
C -> A, B; A->D1, B->D2
"""
references = References()
references[va].add('a.h')
references[vb].add('b.h')
b = Resource(SimpleCell('user/blockb/b.h'))
a = Resource(SimpleCell('user/blocka/a.h'))
d1 = Resource(SimpleCell('user/blockd/d.h'))
if conflict:
d2 = Resource(SimpleCell('user/blockd/d.h', CPP))
else:
d2 = Resource(SimpleCell('user/blockd/d.h'))
a.cell.dependencies.explicit.add(d1.name)
b.cell.dependencies.explicit.add(d2.name)
tables = {va: [vd1], vb: [vd2], vd1: [], vd2: []}
api = FakeApi(zip([va, vb, vd1, vd2], [a, b, d1, d2]), tables)
base_table = BlockVersionTable([vc, va, vb])
biiout = OutputStream()
graph, closure, _ = build_closure(api,
references,
base_table,
biiout=biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va, vb, vd1, vd2])
expected_graph.add_edge(va, vd1)
expected_graph.add_edge(vb, vd2)
self.assertEqual(expected_graph, graph)
self.assertEqual({a.name, b.name, d1.name}, set(closure.keys()))
if conflict:
self.assertIn('Incompatible dependency "user/blockd/d.h"',
str(biiout))
else:
self.assertEqual("", str(biiout))
def simple_collisions_test(self):
g1 = BlockVersionGraph()
g2 = BlockVersionGraph()
brl0 = BRLBlock("user/user/block/master")
v0 = BlockVersion(brl0, 0)
v1 = BlockVersion(brl0, 1)
g1.add_node(v0)
g2.add_node(v1)
expected = BlockVersionGraph()
expected.add_nodes([v0, v1])
self.assertEqual(expected, g1.collision(g2))
self.assertEqual(expected, g2.collision(g1))
def __init__(self):
self.references = None # References()
self.closure = Closure() # {BlockCellName: Resource}
self.src_graph = BlockVersionGraph() # BlockVersionGraph
self.dep_graph = BlockVersionGraph() # BlockVersionGraph
def empty_test(self):
"""When one is empty, it should always be compatible"""
g1 = BlockVersionGraph()
g2 = BlockVersionGraph()
self.assertFalse(g1.collision(g2))
brl0 = BRLBlock("user/user/block/master")
brl1 = BRLBlock("user/user/block2/master")
v0 = BlockVersion(brl0, 0)
v1 = BlockVersion(brl1, 1)
g1.add_node(v0)
self.assertFalse(g1.collision(g2))
self.assertFalse(g2.collision(g1))
g1.add_node(v1)
self.assertFalse(g1.collision(g2))
self.assertFalse(g2.collision(g1))
def build_closure(biiapi, references, base_table, settings=None, biiout=None):
graph = BlockVersionGraph()
closure = Closure()
visited = set()
visited_block_versions = set()
overwrites = defaultdict(set)
if references:
assert not ([r for r in references.explode() if r.block_version.block_name in base_table
and base_table[r.block_version.block_name] != r.block_version])
frontier = [(references, base_table)] # Of tuple (references, base_table)
else:
frontier = []
class Visited(namedtuple('Visited', 'version cell_name, dep_table')):
def __hash__(self):
return hash((self.version, self.cell_name))
while frontier:
references, base_table = frontier.pop()
retrieved = biiapi.get_published_resources(references)
for block_version, resources in retrieved.iteritems():
graph.add_node(block_version)
dep_targets = set()
if block_version not in visited_block_versions:
cell_names = biiapi.get_cells_snapshot(block_version)
_add_implicit_targets(dep_targets, cell_names, block_version)
visited_block_versions.add(block_version)
try:
up_table = biiapi.get_dep_table(block_version)
except Exception as e:
raise BiiException('%s\nbiicode needs compare your local "%s" block with your last'
' version published one. If you want to delete it, delete the '
'folder in the filesystem.' % (str(e),
block_version.block_name))
effective, propagate, overwrite = compute_effective(base_table, up_table,
block_version.block_name)
if overwrite:
overwrites[block_version].update(overwrite)
for cell_name, resource in resources.iteritems():
closure.add_item(resource, block_version, biiout)
visited.add(Visited(block_version, cell_name, base_table))
_update_dep_targets(dep_targets, resource, settings, biiout)
other_refs = References()
self_refs = References()
for target in dep_targets:
if target.block_name == block_version.block_name:
next_visit = Visited(block_version, target.cell_name, base_table)
if next_visit not in visited:
self_refs[block_version].add(target.cell_name)
else:
next_version = effective[target.block_name]
graph.add_edge(block_version, next_version)
if next_version.time is not None:
next_visit = Visited(next_version, target.cell_name, propagate)
if next_visit not in visited:
other_refs[next_version].add(target.cell_name)
if other_refs:
frontier.append((other_refs, propagate))
if self_refs:
frontier.append((self_refs, base_table))
return graph, closure, overwrites
def simple_collisions_test(self):
g1 = BlockVersionGraph()
g2 = BlockVersionGraph()
brl0 = BRLBlock('user/user/block/master')
v0 = BlockVersion(brl0, 0)
v1 = BlockVersion(brl0, 1)
g1.add_node(v0)
g2.add_node(v1)
expected = BlockVersionGraph()
expected.add_nodes([v0, v1])
self.assertEqual(expected, g1.collision(g2))
self.assertEqual(expected, g2.collision(g1))
def diamond_collisions_test(self):
g1 = BlockVersionGraph()
g2 = BlockVersionGraph()
brlA = BRLBlock('user/user/blockA/master')
brlB = BRLBlock('user/user/blockB/master')
brlC = BRLBlock('user/user/blockC/master')
brlD = BRLBlock('user/user/blockD/master')
brlE = BRLBlock('user/user/blockE/master')
brlF = BRLBlock('user/user/blockF/master')
vA0 = BlockVersion(brlA, 0)
vA1 = BlockVersion(brlA, 1)
vB = BlockVersion(brlB, 0)
vC = BlockVersion(brlC, 1)
vD = BlockVersion(brlD, 0)
vE = BlockVersion(brlE, 3)
vF = BlockVersion(brlF, 13)
g1.add_nodes([vA0, vB, vD, vF, vE])
g1.add_edge(vB, vA0)
g1.add_edge(vD, vB)
g1.add_edge(vA0, vE)
g2.add_nodes([vA1, vC, vD, vE])
g2.add_edge(vC, vA1)
g2.add_edge(vD, vC)
g2.add_edge(vA1, vE)
expected = BlockVersionGraph()
expected.add_nodes([vA0, vA1, vB, vC, vD])
expected.add_edge(vC, vA1)
expected.add_edge(vD, vC)
expected.add_edge(vD, vB)
expected.add_edge(vB, vA0)
self.assertEqual(expected, g1.collision(g2))
self.assertEqual(expected, g2.collision(g1))
def compatible_test(self):
'''both not empty, but compatible'''
brl0 = BRLBlock('user/user/block/master')
brl1 = BRLBlock('user/user/block2/master')
v0 = BlockVersion(brl0, 0)
v1 = BlockVersion(brl1, 1)
g1 = BlockVersionGraph()
g2 = BlockVersionGraph()
g1.add_node(v0)
g2.add_node(v1)
self.assertFalse(g1.collision(g2))
self.assertFalse(g2.collision(g1))
g1.add_node(v1)
g2.add_node(v0)
self.assertFalse(g1.collision(g2))
self.assertFalse(g2.collision(g1))
def incompatible_test(self):
'''both not empty, and incompatible'''
brl0 = BRLBlock('user/user/block/master')
brl1 = BRLBlock('user/user/block2/master')
v0 = BlockVersion(brl0, 0)
v1 = BlockVersion(brl1, 1)
v2 = BlockVersion(brl1, 0)
v3 = BlockVersion(brl0, 1)
g1 = BlockVersionGraph()
g2 = BlockVersionGraph()
g1.add_node(v0)
g2.add_node(v3)
self.assertTrue(g1.collision(g2))
self.assertTrue(g2.collision(g1))
g1 = BlockVersionGraph()
g2 = BlockVersionGraph()
g1.add_nodes([v0, v1])
g2.add_nodes([v2, v3])
self.assertTrue(g1.collision(g2))
self.assertTrue(g2.collision(g1))
def empty_test(self):
'''When one is empty, it should always be compatible'''
g1 = BlockVersionGraph()
g2 = BlockVersionGraph()
self.assertFalse(g1.collision(g2))
brl0 = BRLBlock('user/user/block/master')
brl1 = BRLBlock('user/user/block2/master')
v0 = BlockVersion(brl0, 0)
v1 = BlockVersion(brl1, 1)
g1.add_node(v0)
self.assertFalse(g1.collision(g2))
self.assertFalse(g2.collision(g1))
g1.add_node(v1)
self.assertFalse(g1.collision(g2))
self.assertFalse(g2.collision(g1))
def virtual_test(self):
references = References()
references[va].add('a.h')
code = (
'def virtual(settings):\n\tif(settings.os.family == "windows"):return "win"\n'
'\telse: return "nix"')
a = Resource(VirtualCell('user/blocka/a.h', code, {'win', 'nix'}))
awin = Resource(SimpleCell('user/blocka/win/a.h'))
anix = Resource(SimpleCell('user/blocka/nix/a.h'))
b = Resource(SimpleCell('user/blockb/b.h'))
d1 = Resource(SimpleCell('user/blockd/d.h'))
awin.cell.dependencies.explicit.add(b.name)
anix.cell.dependencies.explicit.add(d1.name)
tables = {va: [vb, vd1], vb: [], vd1: []}
api = FakeApi(zip([va, va, va, vb, vd1], [a, awin, anix, b, d1]),
tables)
#With windows settings
settings = Settings(OSInfo(OSFamily("Windows")))
biiout = OutputStream()
graph, closure, _ = build_closure(api, references, {}, settings,
biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va, vb])
expected_graph.add_edge(va, vb)
self.assertEqual(expected_graph, graph)
expected_closure = Closure({
a.name: ClosureItem(a, va),
awin.name: ClosureItem(awin, va),
b.name: ClosureItem(b, vb),
})
self.assertEqual(expected_closure, closure)
self.assertEqual("", str(biiout))
#Change settings
settings = Settings(OSInfo(OSFamily("Linux")))
biiout = OutputStream()
graph, closure, _ = build_closure(api, references, {}, settings,
biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va, vd1])
expected_graph.add_edge(va, vd1)
self.assertEqual(expected_graph, graph)
expected_closure = Closure({
a.name: ClosureItem(a, va),
anix.name: ClosureItem(anix, va),
d1.name: ClosureItem(d1, vd1),
})
self.assertEqual(expected_closure, closure)
self.assertEqual("", str(biiout))
def connected_graphs_no_collisions_test(self):
g1 = BlockVersionGraph()
g2 = BlockVersionGraph()
brl0 = BRLBlock('user/user/block/master')
brl1 = BRLBlock('user/user/block2/master')
v0 = BlockVersion(brl0, 0)
v1 = BlockVersion(brl1, 1)
g1.add_node(v0)
g2.add_node(v0)
self.assertEqual(BlockVersionGraph(), g1.collision(g2))
self.assertEqual(BlockVersionGraph(), g2.collision(g1))
g1.add_node(v1)
self.assertEqual(BlockVersionGraph(), g1.collision(g2))
self.assertEqual(BlockVersionGraph(), g2.collision(g1))
def compatible_test(self):
"""both not empty, but compatible"""
brl0 = BRLBlock("user/user/block/master")
brl1 = BRLBlock("user/user/block2/master")
v0 = BlockVersion(brl0, 0)
v1 = BlockVersion(brl1, 1)
g1 = BlockVersionGraph()
g2 = BlockVersionGraph()
g1.add_node(v0)
g2.add_node(v1)
self.assertFalse(g1.collision(g2))
self.assertFalse(g2.collision(g1))
g1.add_node(v1)
g2.add_node(v0)
self.assertFalse(g1.collision(g2))
self.assertFalse(g2.collision(g1))
def diamond_collisions_test(self):
g1 = BlockVersionGraph()
g2 = BlockVersionGraph()
brlA = BRLBlock("user/user/blockA/master")
brlB = BRLBlock("user/user/blockB/master")
brlC = BRLBlock("user/user/blockC/master")
brlD = BRLBlock("user/user/blockD/master")
brlE = BRLBlock("user/user/blockE/master")
brlF = BRLBlock("user/user/blockF/master")
vA0 = BlockVersion(brlA, 0)
vA1 = BlockVersion(brlA, 1)
vB = BlockVersion(brlB, 0)
vC = BlockVersion(brlC, 1)
vD = BlockVersion(brlD, 0)
vE = BlockVersion(brlE, 3)
vF = BlockVersion(brlF, 13)
g1.add_nodes([vA0, vB, vD, vF, vE])
g1.add_edge(vB, vA0)
g1.add_edge(vD, vB)
g1.add_edge(vA0, vE)
g2.add_nodes([vA1, vC, vD, vE])
g2.add_edge(vC, vA1)
g2.add_edge(vD, vC)
g2.add_edge(vA1, vE)
expected = BlockVersionGraph()
expected.add_nodes([vA0, vA1, vB, vC, vD])
expected.add_edge(vC, vA1)
expected.add_edge(vD, vC)
expected.add_edge(vD, vB)
expected.add_edge(vB, vA0)
self.assertEqual(expected, g1.collision(g2))
self.assertEqual(expected, g2.collision(g1))
def build_closure(biiapi, references, base_table, settings=None, biiout=None):
graph = BlockVersionGraph()
closure = Closure()
visited = set()
visited_block_versions = set()
overwrites = defaultdict(set)
if references:
assert not ([
r for r in references.explode()
if r.block_version.block_name in base_table
and base_table[r.block_version.block_name] != r.block_version
])
frontier = [(references, base_table)
] # Of tuple (references, base_table)
else:
frontier = []
class Visited(namedtuple('Visited', 'version cell_name, dep_table')):
def __hash__(self):
return hash((self.version, self.cell_name))
while frontier:
references, base_table = frontier.pop()
retrieved = biiapi.get_published_resources(references)
for block_version, resources in retrieved.iteritems():
graph.add_node(block_version)
dep_targets = set()
if block_version not in visited_block_versions:
cell_names = biiapi.get_cells_snapshot(block_version)
_add_implicit_targets(dep_targets, cell_names, block_version)
visited_block_versions.add(block_version)
try:
up_table = biiapi.get_dep_table(block_version)
except Exception as e:
raise BiiException(
'%s\nbiicode needs compare your local "%s" block with your last'
' version published one. If you want to delete it, delete the '
'folder in the filesystem.' %
(str(e), block_version.block_name))
effective, propagate, overwrite = compute_effective(
base_table, up_table, block_version.block_name)
if overwrite:
overwrites[block_version].update(overwrite)
for cell_name, resource in resources.iteritems():
closure.add_item(resource, block_version, biiout)
visited.add(Visited(block_version, cell_name, base_table))
_update_dep_targets(dep_targets, resource, settings, biiout)
other_refs = References()
self_refs = References()
for target in dep_targets:
if target.block_name == block_version.block_name:
next_visit = Visited(block_version, target.cell_name,
base_table)
if next_visit not in visited:
self_refs[block_version].add(target.cell_name)
else:
next_version = effective[target.block_name]
graph.add_edge(block_version, next_version)
if next_version.time is not None:
next_visit = Visited(next_version, target.cell_name,
propagate)
if next_visit not in visited:
other_refs[next_version].add(target.cell_name)
if other_refs:
frontier.append((other_refs, propagate))
if self_refs:
frontier.append((self_refs, base_table))
return graph, closure, overwrites
def incompatible_test(self):
"""both not empty, and incompatible"""
brl0 = BRLBlock("user/user/block/master")
brl1 = BRLBlock("user/user/block2/master")
v0 = BlockVersion(brl0, 0)
v1 = BlockVersion(brl1, 1)
v2 = BlockVersion(brl1, 0)
v3 = BlockVersion(brl0, 1)
g1 = BlockVersionGraph()
g2 = BlockVersionGraph()
g1.add_node(v0)
g2.add_node(v3)
self.assertTrue(g1.collision(g2))
self.assertTrue(g2.collision(g1))
g1 = BlockVersionGraph()
g2 = BlockVersionGraph()
g1.add_nodes([v0, v1])
g2.add_nodes([v2, v3])
self.assertTrue(g1.collision(g2))
self.assertTrue(g2.collision(g1))
