def __init__(self, references=None):
self._elements = {
} # {rootID: (cell_id, content_id), block_version, name }
#The set of references that have to be explored yet, initially, the input references
self.frontier = references or References()
#Those references that were impossible to fetch, due to permissions or deletions
self.broken = References()
def test_basic(self):
store = TestingMemServerStore()
brl_user = '******'
store.create_user(User(brl_user))
brl_block = BRLBlock('user/user/block/master')
publisher = TestPublisher(brl_user, store)
publisher.publish(
brl_block, {
'r1.h': ('r1', ['user/block/r2.h']),
'r2.h': ('r2', []),
'r3.h': ('r3', [])
})
store = MemServerStore(store)
translator = ReferenceTranslatorService(store, brl_user)
ver = BlockVersion(brl_block, 0)
missing = References()
missing[ver].add('r1.h')
closure = CompatibilityClosure(missing)
full_graph = BlockVersionGraph()
full_graph.add_nodes([ver])
build_compatibility_closure(translator, closure, {ver}, full_graph)
self.assertEqual(References(), closure.broken)
expected = set()
expected.add('user/block/r1.h')
expected.add('user/block/r2.h')
self.assertEqual(expected, closure.block_cell_names)
def test_references(self):
r = References()
bv3 = BlockVersion(BRLBlock('user/user/block/master'), 3)
bv4 = BlockVersion(BRLBlock('user/user/block/master'), 4)
cn0 = CellName('foo.h')
cn1 = CellName('foo1.h')
r[bv3].add(cn0)
r[bv3].add(cn1)
r[bv4].add(cn0)
l = r.explode()
self.assertEqual({(bv3, cn0), (bv3, cn1), (bv4, cn0)}, set(l))
def simple_test(self):
""" Blocks: C
Deps: A, B
C -> A -> B
"""
references = References()
references[va].add('a.h')
base_table = BlockVersionTable(
[vc, va]) # The result including or excluding va is same
b = Resource(SimpleCell('user/blockb/b.h'))
a = Resource(SimpleCell('user/blocka/a.h'))
a2 = Resource(SimpleCell('user/blocka/a2.h'))
a.cell.dependencies.explicit.add(a2.name)
a2.cell.dependencies.explicit.add(b.name)
tables = {va: [vb], vb: []}
api = FakeApi(zip([va, va, vb], [a, a2, b]), tables)
biiout = OutputStream()
graph, closure, _ = build_closure(api,
references,
base_table,
biiout=biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va, vb])
expected_graph.add_edge(va, vb)
self.assertEqual(expected_graph, graph)
expected_closure = Closure({
a.name: ClosureItem(a, va),
a2.name: ClosureItem(a2, va),
b.name: ClosureItem(b, vb)
})
self.assertEqual(expected_closure, closure)
self.assertEqual("", str(biiout))
def test_interface(self):
cell_id1 = ID((0, 0, 0))
content_id1 = ID((0, 0, 0))
cell_id2 = ID((0, 1, 0))
content_id2 = ID((0, 1, 0))
brl0 = BRLBlock('user/user/block/master')
brl1 = BRLBlock('user/user/block2/master')
v0 = BlockVersion(brl0, 0)
v1 = BlockVersion(brl0, 1)
v2 = BlockVersion(brl1, 0)
c1 = CompatibilityClosure()
c1.add_item((cell_id1, content_id1), cell_id1, v0, 'file1')
self.assertEqual("{0:0:0: ((0:0:0, 0:0:0), user/block: 0, 'file1')}", str(c1))
# adding a compatible item is no problem
c1.add_item((cell_id1, content_id1), cell_id1, v1, 'file1')
self.assertEqual("{0:0:0: ((0:0:0, 0:0:0), user/block: 1, 'file1')}", str(c1))
c1.add_item((cell_id2, content_id2), cell_id2, v2, 'file2')
self.assertIn("0:0:0: ((0:0:0, 0:0:0), user/block: 1, 'file1')", str(c1))
self.assertIn("0:1:0: ((0:1:0, 0:1:0), user/block2: 0, 'file2')", str(c1))
self.assertEqual({v0.block_name + 'file1', v2.block_name + 'file2'}, c1.block_cell_names)
refs = References()
refs[v1].add('file1')
refs[v2].add('file2')
self.assertEqual(refs, c1.references)
def overwrite_test(self):
""" Blocks: C (defines in requirements B2)
Deps: A, B2
C -> A -> B1
"""
references = References()
references[va].add('a.h')
b = Resource(SimpleCell('user/blockb/b.h'))
b2 = Resource(SimpleCell('user/blockb/b.h'), CPP)
a = Resource(SimpleCell('user/blocka/a.h'))
a2 = Resource(SimpleCell('user/blocka/a2.h'))
a.cell.dependencies.explicit.add(a2.name)
a2.cell.dependencies.explicit.add(b.name)
tables = {va: [vb], vb: [], vb2: []}
api = FakeApi(zip([va, va, vb, vb2], [a, a2, b, b2]), tables)
base_table = BlockVersionTable([vc, va, vb2]) # Note B2 defined here
biiout = OutputStream()
graph, closure, _ = build_closure(api,
references,
base_table,
biiout=biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va, vb2])
expected_graph.add_edge(va, vb2)
self.assertEqual(expected_graph, graph)
expected_closure = Closure({
a.name: ClosureItem(a, va),
a2.name: ClosureItem(a2, va),
b.name: ClosureItem(b2, vb2)
})
self.assertEqual(expected_closure, closure)
self.assertEqual("", str(biiout))
def dep_overwriten_in_blocks_test(self):
""" Blocks: C, B (None version)
Deps: A
C -> A -> B
"""
references = References()
references[va].add('a.h')
vbn = BlockVersion(vb.block, None)
b = Resource(SimpleCell('user/blockb/b.h'))
a = Resource(SimpleCell('user/blocka/a.h'))
a2 = Resource(SimpleCell('user/blocka/a2.h'))
a.cell.dependencies.explicit.add(a2.name)
a2.cell.dependencies.explicit.add(b.name)
tables = {va: [vb], vb: []}
api = FakeApi(zip([va, va, vb], [a, a2, b]), tables)
base_table = BlockVersionTable([vc, vbn])
biiout = OutputStream()
graph, closure, _ = build_closure(api,
references,
base_table,
biiout=biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va])
expected_graph.add_edge(va, vbn)
self.assertEqual(expected_graph, graph)
expected_closure = Closure({
a.name: ClosureItem(a, va),
a2.name: ClosureItem(a2, va)
})
self.assertEqual(expected_closure, closure)
self.assertEqual("", str(biiout))
def test_build_closure_different_versions_restricted(self):
'''Computes a CompatibilityClosure in which two versions of blockA actually point to the
same unmodified resource with the typical diamond layout
Also computes and checks the BlockClosure for that layout'''
ref_translator = Mock()
depA1 = BlockVersion(BRLBlock('user/user/blockA/branch'), 4)
depA2 = BlockVersion(BRLBlock('user/user/blockA/branch'), 5)
baseB = BlockVersion(BRLBlock('user/user/blockB/branch'), 2)
baseC = BlockVersion(BRLBlock('user/user/blockC/branch'), 3)
full_graph = BlockVersionGraph()
full_graph.add_nodes([baseB, baseC, depA1, depA2])
full_graph.add_edge(baseB, depA1)
full_graph.add_edge(baseC, depA2)
def res_method(*args):
depsb = [BlockCellName('user/blockA/a.h')]
depsc = [BlockCellName('user/blockA/a.h')]
result = ReferencedResources()
for ref in args[0].explode():
result[ref.block_version][ref.ref] = \
{Reference(depA1, 'a.h'): ((0, 0), 0, []),
Reference(depA2, 'a.h'): ((0, 0), 0, []),
Reference(baseB, 'b.h'): ((1, 4), 1, depsb),
Reference(baseC, 'c.h'): ((2, 3), 2, depsc)}[ref]
return result
ref_translator.get_published_min_refs.side_effect = res_method
missing = References()
missing.add(Reference(baseB, 'b.h'))
missing.add(Reference(baseC, 'c.h'))
closure = CompatibilityClosure(missing)
build_compatibility_closure(ref_translator, closure, {baseC}, full_graph)
self.assertEqual(References(), closure.broken)
self.assertEqual({BlockCellName('user/blockC/c.h')}, closure.block_cell_names)
#self.assertIn(Reference(depA1, 'a.h'), closure.references)
self.assertNotIn(baseB, closure.references)
self.assertNotIn(depA1, closure.references)
self.assertNotIn(depA2, closure.references)
self.assertIn(baseC, closure.references)
expected_frontier = References()
expected_frontier[baseB].add('b.h')
expected_frontier[depA2].add('a.h')
self.assertEqual(expected_frontier, closure.frontier)
def _get_not_found_refs(requested_refs, found_refs):
not_found_refs = References()
for block_version, cell_names in requested_refs.iteritems():
version_resources = found_refs.get(block_version, {})
missing = cell_names.difference(version_resources)
if missing:
not_found_refs[block_version] = missing
return not_found_refs
def closure(self): # Lazy computation of closures
if self._closure is None:
missing = References()
for targets in self.dep_dict.values():
for target in targets:
missing[self.block_version].add(target.cell_name)
self._closure = CompatibilityClosure(missing)
return self._closure
def test_virtual_no_settings(self):
'''builds a very simple closure in which virtual cells are involved. Maybe it is not very
useful, as now virtuality is not handled at this level, only at the MemServerStore level,
so this test might be redundant'''
ref_translator, depA, depB, baseC, full_graph = self._virtual_setup()
missing = References()
missing.add(Reference(baseC, 'c.h'))
closure = CompatibilityClosure(missing)
build_compatibility_closure(ref_translator, closure, full_graph.nodes, full_graph)
self.assertEqual(References(), closure.broken)
self.assertEqual(set([BlockCellName('user/blockA/a.h'),
BlockCellName('user/blockB/b.h'),
BlockCellName('user/blockC/c.h'),
BlockCellName('user/blockC/win/c.h'),
BlockCellName('user/blockC/nix/c.h')]),
closure.block_cell_names)
def build_compatibility_closure(api, closure, restricted_versions, full_graph):
""" Builds compatibility closure for references
Args:
api (biicode.server.reference_translator.reference_translator_service.\
ReferenceTranslatorService):
MUST implement get_dep_table and get_published_min_refs
references: missing references to fetch and add to closure
"""
retrieved = References() # Accumulates all references retrieved
# define the frontier to be taken into account to expand, and update it removing those elements
frontier = References()
for version, cell_names in closure.frontier.items():
if version in restricted_versions:
frontier[version] = cell_names
del closure.frontier[version]
while frontier:
#logger.debug("Missing to fetch: %s" % str(references))
min_cells = api.get_published_min_refs(frontier)
# min_cells are {block_version: {cell_name: ((cell_id, content_id), root_id, deps)}
#From what has been retrieved, add to closure, and store in missing_dict all the references
# to be translated
missing_dict = References()
for block_version, cell_dict in min_cells.iteritems():
for cell_name, ((cell_id, content_id), root_id,
deps) in cell_dict.iteritems():
retrieved[block_version].add(cell_name)
closure.add_item((cell_id, content_id), root_id, block_version,
cell_name)
if deps:
missing_dict[block_version].update(deps)
# Account for deletions, if something was not obtained from api (deleted or changed
# permissions) it will be a missing reference in our closure
for version, names in frontier.iteritems():
broken_names = names.difference(min_cells.get(version, []))
if broken_names:
closure.broken[version].update(broken_names)
# now, the references are the new ones
frontier = _compute_frontier(missing_dict, restricted_versions,
full_graph, retrieved, closure.frontier)
def test_build_closure_different_versions(self):
'''Computes a CompatibilityClosure in which two versions of blockA actually point to the
same unmodified resource with the typical diamond layout
Also computes and checks the BlockClosure for that layout'''
ref_translator = Mock()
depA1 = BlockVersion(BRLBlock('user/user/blockA/branch'), 4)
depA2 = BlockVersion(BRLBlock('user/user/blockA/branch'), 5)
baseB = BlockVersion(BRLBlock('user/user/blockB/branch'), 2)
baseC = BlockVersion(BRLBlock('user/user/blockC/branch'), 3)
full_graph = BlockVersionGraph()
full_graph.add_nodes([baseB, baseC, depA1, depA2])
full_graph.add_edge(baseB, depA1)
full_graph.add_edge(baseC, depA2)
def res_method(*args):
depsb = [BlockCellName('user/blockA/a.h')]
depsc = [BlockCellName('user/blockA/a.h')]
result = ReferencedResources()
for ref in args[0].explode():
result[ref.block_version][ref.ref] = \
{Reference(depA1, 'a.h'): ((0, 0), 0, []),
Reference(depA2, 'a.h'): ((0, 0), 0, []),
Reference(baseB, 'b.h'): ((1, 4), 1, depsb),
Reference(baseC, 'c.h'): ((2, 3), 2, depsc)}[ref]
return result
ref_translator.get_published_min_refs.side_effect = res_method
missing = References()
missing.add(Reference(baseB, 'b.h'))
missing.add(Reference(baseC, 'c.h'))
closure = CompatibilityClosure(missing)
build_compatibility_closure(ref_translator, closure, full_graph.nodes, full_graph)
self.assertEqual(References(), closure.broken)
self.assertEqual(set([BlockCellName('user/blockA/a.h'),
BlockCellName('user/blockB/b.h'),
BlockCellName('user/blockC/c.h')]),
closure.block_cell_names)
#self.assertIn(Reference(depA1, 'a.h'), closure.references)
self.assertIn(baseB, closure.references)
self.assertIn(baseC, closure.references)
def get_block_holder(self, block_version):
""""
return: BlockHolder
"""
assert block_version.time is not None
refs = References()
block_cells_name = self.get_cells_snapshot(block_version)
refs[block_version] = set(block_cells_name)
resources = self.get_published_resources(refs)
return BlockHolder(block_version.block_name, resources[block_version])
def build_compatibility_closure(api, closure, restricted_versions, full_graph):
""" Builds compatibility closure for references
Args:
api (biicode.server.reference_translator.reference_translator_service.\
ReferenceTranslatorService):
MUST implement get_dep_table and get_published_min_refs
references: missing references to fetch and add to closure
"""
retrieved = References() # Accumulates all references retrieved
# define the frontier to be taken into account to expand, and update it removing those elements
frontier = References()
for version, cell_names in closure.frontier.items():
if version in restricted_versions:
frontier[version] = cell_names
del closure.frontier[version]
while frontier:
#logger.debug("Missing to fetch: %s" % str(references))
min_cells = api.get_published_min_refs(frontier)
# min_cells are {block_version: {cell_name: ((cell_id, content_id), root_id, deps)}
#From what has been retrieved, add to closure, and store in missing_dict all the references
# to be translated
missing_dict = References()
for block_version, cell_dict in min_cells.iteritems():
for cell_name, ((cell_id, content_id), root_id, deps) in cell_dict.iteritems():
retrieved[block_version].add(cell_name)
closure.add_item((cell_id, content_id), root_id, block_version, cell_name)
if deps:
missing_dict[block_version].update(deps)
# Account for deletions, if something was not obtained from api (deleted or changed
# permissions) it will be a missing reference in our closure
for version, names in frontier.iteritems():
broken_names = names.difference(min_cells.get(version, []))
if broken_names:
closure.broken[version].update(broken_names)
# now, the references are the new ones
frontier = _compute_frontier(missing_dict, restricted_versions, full_graph, retrieved,
closure.frontier)
def deserialize(data):
try:
res = HiveDependencies()
res.dep_table = BlockVersionTable.deserialize(data[HiveDependencies.SERIAL_DEP_TABLE])
res.references = References.deserialize(data[HiveDependencies.SERIAL_REFERENCES])
res.closure = Closure.deserialize(data[HiveDependencies.SERIAL_CLOSURE])
res.src_graph = BlockVersionGraph.deserialize(data[HiveDependencies.SERIAL_SRC_GRAPH])
res.dep_graph = BlockVersionGraph.deserialize(data[HiveDependencies.SERIAL_DEP_GRAPH])
return res
except Exception as e:
raise BiiSerializationException(e)
def test_cached_references(self):
s = References()
s[self.block_version] = {CellName("alf.c"), CellName("willy.c")}
self.proxy.get_published_resources(s)
self.assertTrue(self.restapi.get_published_resources.called)
self.restapi.get_published_resources.called = False
c = self.proxy.get_published_resources(s)
self.assertFalse(self.restapi.get_published_resources.called)
self.assertEqual(c, self.referenced_resources)
def virtual_test(self):
references = References()
references[va].add('a.h')
code = (
'def virtual(settings):\n\tif(settings.os.family == "windows"):return "win"\n'
'\telse: return "nix"')
a = Resource(VirtualCell('user/blocka/a.h', code, {'win', 'nix'}))
awin = Resource(SimpleCell('user/blocka/win/a.h'))
anix = Resource(SimpleCell('user/blocka/nix/a.h'))
b = Resource(SimpleCell('user/blockb/b.h'))
d1 = Resource(SimpleCell('user/blockd/d.h'))
awin.cell.dependencies.explicit.add(b.name)
anix.cell.dependencies.explicit.add(d1.name)
tables = {va: [vb, vd1], vb: [], vd1: []}
api = FakeApi(zip([va, va, va, vb, vd1], [a, awin, anix, b, d1]),
tables)
#With windows settings
settings = Settings(OSInfo(OSFamily("Windows")))
biiout = OutputStream()
graph, closure, _ = build_closure(api, references, {}, settings,
biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va, vb])
expected_graph.add_edge(va, vb)
self.assertEqual(expected_graph, graph)
expected_closure = Closure({
a.name: ClosureItem(a, va),
awin.name: ClosureItem(awin, va),
b.name: ClosureItem(b, vb),
})
self.assertEqual(expected_closure, closure)
self.assertEqual("", str(biiout))
#Change settings
settings = Settings(OSInfo(OSFamily("Linux")))
biiout = OutputStream()
graph, closure, _ = build_closure(api, references, {}, settings,
biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va, vd1])
expected_graph.add_edge(va, vd1)
self.assertEqual(expected_graph, graph)
expected_closure = Closure({
a.name: ClosureItem(a, va),
anix.name: ClosureItem(anix, va),
d1.name: ClosureItem(d1, vd1),
})
self.assertEqual(expected_closure, closure)
self.assertEqual("", str(biiout))
def test_broken_closure(self):
'''computes a closure in which cell blockB/b.h depends on blockA/a.h, but that a.h
is not found. The algorithms of closures must return that as a missing or broken dependency
'''
ref_translator = Mock()
baseB = BlockVersion(BRLBlock('user/user/blockB/branch'), 2)
depA1 = BlockVersion(BRLBlock('user/user/blockA/branch'), 4)
full_graph = BlockVersionGraph()
full_graph.add_nodes([baseB, depA1])
full_graph.add_edge(baseB, depA1)
def res_method(*args):
depsb = [BlockCellName('user/blockA/a.h')]
result = ReferencedResources()
for ref in args[0].explode():
try:
result[ref.block_version][ref.ref] = \
{
Reference(baseB, 'b.h'): ((0, 0), 0, depsb),
}[ref]
except KeyError:
pass
return result
ref_translator.get_published_min_refs.side_effect = res_method
missing = References()
missing.add(Reference(baseB, 'b.h'))
closure = CompatibilityClosure(missing)
build_compatibility_closure(ref_translator, closure, full_graph.nodes, full_graph)
self.assertEqual({BlockCellName('user/blockB/b.h')}, closure.block_cell_names)
self.assertIn(baseB, closure.references)
expected_missing = References()
expected_missing[depA1].add('a.h')
self.assertEqual(closure.broken, expected_missing)
def assert_published_resources(self, api_manager, version, files):
"""
:param version: BlockVersion published
:files: dict of {path_to_file: str_bytes}"""
cellnames = {blockcellname.cell_name: content
for blockcellname, content in files.iteritems()}
refs = References()
refs[version] = set(cellnames.keys())
referenced_resources = api_manager.get_published_resources(refs)
for thefile, thecontent in cellnames.iteritems():
resource = referenced_resources[version][thefile]
self.assertIsInstance(resource, Resource)
self.assertEqual(resource.content.load.text, thecontent)
def test_performance_depth(self):
brl_user = '******'
store = MongoServerStore(self.conn, self.__class__.__name__)
store.create_user(User(brl_user))
publisher = TestPublisher(brl_user, store)
brl_block = BRLBlock('user/user/block/master')
count = 500
resource_info = {
'cell%d.h' % i:
('content %d' % i, ['user/block/cell%d.h' % (i - 1)] if i else [])
for i in range(count)
}
publisher.publish(brl_block, resource_info)
timer = self.get_timer()
start_time = timer()
# prof = cProfile.Profile()
# prof.enable()
store = MemServerStore(store)
# print 'MEMSTORE DEPTH SIZE 0', asizeof(store) / 1000000.0
translator = ReferenceTranslatorService(store, brl_user)
version = BlockVersion(brl_block, 0)
missing = References()
missing[version].add('cell%d.h' % (count - 1))
closure = CompatibilityClosure(missing)
full_graph = BlockVersionGraph()
full_graph.add_nodes([version])
build_compatibility_closure(translator, closure, [version], full_graph)
self.assertEqual(References(), closure.broken)
#print 'CLOSURE SIZE ', asizeof(closure) / 1000000.0
#print 'MEMSTORE SIZE ', asizeof(store) / 1000000.0
elapsed_time = timer() - start_time
#print 'Closure time', elapsed_time
self.assertEqual({brl_block.block_name + c
for c in resource_info}, closure.block_cell_names)
self.assertLess(elapsed_time, 5)
def _compute_frontier(missing_dict, restricted_versions, full_graph, retrieved,
open_frontier):
frontier = References()
for block_version, targets in missing_dict.iteritems():
neighbours = BlockVersionTable(full_graph.neighbours(block_version))
for target in targets:
if target.block_name != block_version.block_name:
other_version = neighbours.get(target.block_name)
if other_version not in restricted_versions:
open_frontier[other_version].add(target.cell_name)
else:
other_version = block_version
if other_version in restricted_versions and target.cell_name not in retrieved[
other_version]:
frontier[other_version].add(target.cell_name)
return frontier
def deserialize(data):
try:
res = HiveDependencies()
res.dep_table = BlockVersionTable.deserialize(
data[HiveDependencies.SERIAL_DEP_TABLE])
res.references = References.deserialize(
data[HiveDependencies.SERIAL_REFERENCES])
res.closure = Closure.deserialize(
data[HiveDependencies.SERIAL_CLOSURE])
res.src_graph = BlockVersionGraph.deserialize(
data[HiveDependencies.SERIAL_SRC_GRAPH])
res.dep_graph = BlockVersionGraph.deserialize(
data[HiveDependencies.SERIAL_DEP_GRAPH])
return res
except Exception as e:
raise BiiSerializationException(e)
def test_performance_breadth(self):
store = MongoServerStore(self.conn, self.__class__.__name__)
store.create_user(User("user2"))
publisher = TestPublisher("user2", store)
brl_block = BRLBlock('user2/user2/block/master')
count = 1000
resource_info = {}
for i in xrange(count):
deps = DependencySet()
if i > 0:
deps = DependencySet()
for j in range(max(0, i - 25), i):
deps.explicit.add(BlockCellName('user2/block/cell%d.h' %
j))
deps.resolved.add(
CPPDeclaration('user2/block/cell%d.h' % j))
deps.unresolved.add(CPPDeclaration('path/to/file.h'))
deps.implicit.add(BlockCellName('user2/block/cell%d.h' % j))
resource_info['cell%d.h' % i] = 'content %d' % i, deps
publisher.publish(brl_block, resource_info)
timer = self.get_timer()
start_time = timer()
store = MemServerStore(store)
#print 'MEMSTORE SIZE 0', asizeof(store) / 1000000.0
translator = ReferenceTranslatorService(store, "user2")
version = BlockVersion(brl_block, 0)
missing = References()
missing[version].add('cell%d.h' % (count - 1))
closure = CompatibilityClosure(missing)
full_graph = BlockVersionGraph()
full_graph.add_nodes([version])
build_compatibility_closure(translator, closure, [version], full_graph)
elapsed_time = timer() - start_time
#print 'Closure time', elapsed_time
#print 'CLOSURE SIZE ', asizeof(closure) / 1000000.0
#print 'MEMSTORE SIZE ', asizeof(store) / 1000000.0
# print 'MINCELLS SIZE ', asizeof(store.min_cells)/1000000.0
self.assertEqual({brl_block.block_name + c
for c in resource_info}, closure.block_cell_names)
self.assertLess(elapsed_time, 7)
def test_store_published_resources(self):
s = References()
brl_block = BRLBlock('dummy/dummy/block/master')
block_version = BlockVersion(brl_block, 0)
s[block_version] = [
CellName("alf.c"),
CellName("willy.c"),
CellName('maya.h'),
CellName('win/maya.h'),
CellName('nix/maya.h')
]
alf = Resource(SimpleCell("dummy/block/alf.c"),
Content(ID((0, 1, 2)), Blob("Hello Alf")))
alf.cell.ID = ID((0, 1, 2))
willy = Resource(SimpleCell("dummy/block/willy.c"),
Content(ID((0, 1, 3)), Blob("Hello Willy")))
willy.cell.ID = ID((0, 1, 45))
maya_v = Resource(VirtualCell("dummy/block/maya.h"), None)
maya_v.cell.ID = ID((0, 1, 3))
maya_win = Resource(SimpleCell("dummy/block/win/maya.h"),
Content(ID((0, 1, 4)), Blob("Hello Maya")))
maya_win.cell.ID = ID((0, 1, 4))
maya_nix = Resource(SimpleCell("dummy/block/nix/maya.h"),
Content(ID((0, 1, 5)), Blob("Hello Maya")))
maya_nix.cell.ID = ID((0, 1, 5))
# Expected return
referenced_resources = ReferencedResources()
referenced_resources[block_version].update({
CellName("alf.c"):
alf,
CellName("willy.c"):
willy,
CellName('maya.h'):
maya_v,
CellName('win/maya.h'):
maya_win,
CellName('nix/maya.h'):
maya_nix,
})
self.db.create_published_resources(referenced_resources)
retrieved = self.db.get_published_resources(s)
self.assertEquals(referenced_resources, retrieved)
def compute_src_graph(hive_holder, common_table):
""" computes just the src part of the full version graph.
Side effect: updates requirements of blocks to actually point to real dep versions
"""
graph = BlockVersionGraph()
versions = hive_holder.versions
graph.add_nodes(versions.itervalues())
references = References()
for block_holder in hive_holder.block_holders:
dep_table = block_holder.requirements
base_version = versions[block_holder.block_name]
for target_bcn in block_holder.external_targets():
target_block_name = target_bcn.block_name
if target_block_name in versions:
other_version = versions[target_block_name]
else:
other_version = common_table[target_block_name]
references[other_version].add(target_bcn.cell_name)
graph.add_edge(base_version, other_version)
dep_table.add_version(other_version)
return graph, references
def diamond_test(self, conflict):
""" Blocks: C
Deps: A, B, D1-D2
C -> A, B; A->D1, B->D2
"""
references = References()
references[va].add('a.h')
references[vb].add('b.h')
b = Resource(SimpleCell('user/blockb/b.h'))
a = Resource(SimpleCell('user/blocka/a.h'))
d1 = Resource(SimpleCell('user/blockd/d.h'))
if conflict:
d2 = Resource(SimpleCell('user/blockd/d.h', CPP))
else:
d2 = Resource(SimpleCell('user/blockd/d.h'))
a.cell.dependencies.explicit.add(d1.name)
b.cell.dependencies.explicit.add(d2.name)
tables = {va: [vd1], vb: [vd2], vd1: [], vd2: []}
api = FakeApi(zip([va, vb, vd1, vd2], [a, b, d1, d2]), tables)
base_table = BlockVersionTable([vc, va, vb])
biiout = OutputStream()
graph, closure, _ = build_closure(api,
references,
base_table,
biiout=biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va, vb, vd1, vd2])
expected_graph.add_edge(va, vd1)
expected_graph.add_edge(vb, vd2)
self.assertEqual(expected_graph, graph)
self.assertEqual({a.name, b.name, d1.name}, set(closure.keys()))
if conflict:
self.assertIn('Incompatible dependency "user/blockd/d.h"',
str(biiout))
else:
self.assertEqual("", str(biiout))
def build_closure(biiapi, references, base_table, settings=None, biiout=None):
graph = BlockVersionGraph()
closure = Closure()
visited = set()
visited_block_versions = set()
overwrites = defaultdict(set)
if references:
assert not ([
r for r in references.explode()
if r.block_version.block_name in base_table
and base_table[r.block_version.block_name] != r.block_version
])
frontier = [(references, base_table)
] # Of tuple (references, base_table)
else:
frontier = []
class Visited(namedtuple('Visited', 'version cell_name, dep_table')):
def __hash__(self):
return hash((self.version, self.cell_name))
while frontier:
references, base_table = frontier.pop()
retrieved = biiapi.get_published_resources(references)
for block_version, resources in retrieved.iteritems():
graph.add_node(block_version)
dep_targets = set()
if block_version not in visited_block_versions:
cell_names = biiapi.get_cells_snapshot(block_version)
_add_implicit_targets(dep_targets, cell_names, block_version)
visited_block_versions.add(block_version)
try:
up_table = biiapi.get_dep_table(block_version)
except Exception as e:
raise BiiException(
'%s\nbiicode needs compare your local "%s" block with your last'
' version published one. If you want to delete it, delete the '
'folder in the filesystem.' %
(str(e), block_version.block_name))
effective, propagate, overwrite = compute_effective(
base_table, up_table, block_version.block_name)
if overwrite:
overwrites[block_version].update(overwrite)
for cell_name, resource in resources.iteritems():
closure.add_item(resource, block_version, biiout)
visited.add(Visited(block_version, cell_name, base_table))
_update_dep_targets(dep_targets, resource, settings, biiout)
other_refs = References()
self_refs = References()
for target in dep_targets:
if target.block_name == block_version.block_name:
next_visit = Visited(block_version, target.cell_name,
base_table)
if next_visit not in visited:
self_refs[block_version].add(target.cell_name)
else:
next_version = effective[target.block_name]
graph.add_edge(block_version, next_version)
if next_version.time is not None:
next_visit = Visited(next_version, target.cell_name,
propagate)
if next_visit not in visited:
other_refs[next_version].add(target.cell_name)
if other_refs:
frontier.append((other_refs, propagate))
if self_refs:
frontier.append((self_refs, base_table))
return graph, closure, overwrites
def get_published_resources(auth_user, bson_data):
service = BiiService(app.store, auth_user)
references = References.deserialize(bson_data["data"])
return service.get_published_resources(references)
def references(self):
r = References()
for _, block_version, name in self._elements.itervalues():
r[block_version].add(name)
return r
def test_deepcopy(self):
r = References()
r[BlockVersion(BRLBlock('user/user/block/branch'),
3)].add(CellName('f1.h'))
r2 = copy.deepcopy(r)
self.assertEqual(r, r2)