def virtual_test(self):
references = References()
references[va].add('a.h')
code = (
'def virtual(settings):\n\tif(settings.os.family == "windows"):return "win"\n'
'\telse: return "nix"')
a = Resource(VirtualCell('user/blocka/a.h', code, {'win', 'nix'}))
awin = Resource(SimpleCell('user/blocka/win/a.h'))
anix = Resource(SimpleCell('user/blocka/nix/a.h'))
b = Resource(SimpleCell('user/blockb/b.h'))
d1 = Resource(SimpleCell('user/blockd/d.h'))
awin.cell.dependencies.explicit.add(b.name)
anix.cell.dependencies.explicit.add(d1.name)
tables = {va: [vb, vd1], vb: [], vd1: []}
api = FakeApi(zip([va, va, va, vb, vd1], [a, awin, anix, b, d1]),
tables)
#With windows settings
settings = Settings(OSInfo(OSFamily("Windows")))
biiout = OutputStream()
graph, closure, _ = build_closure(api, references, {}, settings,
biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va, vb])
expected_graph.add_edge(va, vb)
self.assertEqual(expected_graph, graph)
expected_closure = Closure({
a.name: ClosureItem(a, va),
awin.name: ClosureItem(awin, va),
b.name: ClosureItem(b, vb),
})
self.assertEqual(expected_closure, closure)
self.assertEqual("", str(biiout))
#Change settings
settings = Settings(OSInfo(OSFamily("Linux")))
biiout = OutputStream()
graph, closure, _ = build_closure(api, references, {}, settings,
biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va, vd1])
expected_graph.add_edge(va, vd1)
self.assertEqual(expected_graph, graph)
expected_closure = Closure({
a.name: ClosureItem(a, va),
anix.name: ClosureItem(anix, va),
d1.name: ClosureItem(d1, vd1),
})
self.assertEqual(expected_closure, closure)
self.assertEqual("", str(biiout))
def __init__(self):
self.dep_table = None # BlockVersionTable()
self.references = None # References()
# TODO: The whole closure is being serialized. Check performance, improve
self.closure = Closure() # {BlockCellName: Resource}
self.src_graph = BlockVersionGraph() # BlockVersionGraph
self.dep_graph = BlockVersionGraph() # BlockVersionGraph
def overwrite_test(self):
""" Blocks: C (defines in requirements B2)
Deps: A, B2
C -> A -> B1
"""
references = References()
references[va].add('a.h')
b = Resource(SimpleCell('user/blockb/b.h'))
b2 = Resource(SimpleCell('user/blockb/b.h'), CPP)
a = Resource(SimpleCell('user/blocka/a.h'))
a2 = Resource(SimpleCell('user/blocka/a2.h'))
a.cell.dependencies.explicit.add(a2.name)
a2.cell.dependencies.explicit.add(b.name)
tables = {va: [vb], vb: [], vb2: []}
api = FakeApi(zip([va, va, vb, vb2], [a, a2, b, b2]), tables)
base_table = BlockVersionTable([vc, va, vb2]) # Note B2 defined here
biiout = OutputStream()
graph, closure, _ = build_closure(api,
references,
base_table,
biiout=biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va, vb2])
expected_graph.add_edge(va, vb2)
self.assertEqual(expected_graph, graph)
expected_closure = Closure({
a.name: ClosureItem(a, va),
a2.name: ClosureItem(a2, va),
b.name: ClosureItem(b2, vb2)
})
self.assertEqual(expected_closure, closure)
self.assertEqual("", str(biiout))
def simple_test(self):
""" Blocks: C
Deps: A, B
C -> A -> B
"""
references = References()
references[va].add('a.h')
base_table = BlockVersionTable(
[vc, va]) # The result including or excluding va is same
b = Resource(SimpleCell('user/blockb/b.h'))
a = Resource(SimpleCell('user/blocka/a.h'))
a2 = Resource(SimpleCell('user/blocka/a2.h'))
a.cell.dependencies.explicit.add(a2.name)
a2.cell.dependencies.explicit.add(b.name)
tables = {va: [vb], vb: []}
api = FakeApi(zip([va, va, vb], [a, a2, b]), tables)
biiout = OutputStream()
graph, closure, _ = build_closure(api,
references,
base_table,
biiout=biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va, vb])
expected_graph.add_edge(va, vb)
self.assertEqual(expected_graph, graph)
expected_closure = Closure({
a.name: ClosureItem(a, va),
a2.name: ClosureItem(a2, va),
b.name: ClosureItem(b, vb)
})
self.assertEqual(expected_closure, closure)
self.assertEqual("", str(biiout))
def dep_overwriten_in_blocks_test(self):
""" Blocks: C, B (None version)
Deps: A
C -> A -> B
"""
references = References()
references[va].add('a.h')
vbn = BlockVersion(vb.block, None)
b = Resource(SimpleCell('user/blockb/b.h'))
a = Resource(SimpleCell('user/blocka/a.h'))
a2 = Resource(SimpleCell('user/blocka/a2.h'))
a.cell.dependencies.explicit.add(a2.name)
a2.cell.dependencies.explicit.add(b.name)
tables = {va: [vb], vb: []}
api = FakeApi(zip([va, va, vb], [a, a2, b]), tables)
base_table = BlockVersionTable([vc, vbn])
biiout = OutputStream()
graph, closure, _ = build_closure(api,
references,
base_table,
biiout=biiout)
expected_graph = BlockVersionGraph()
expected_graph.add_nodes([va])
expected_graph.add_edge(va, vbn)
self.assertEqual(expected_graph, graph)
expected_closure = Closure({
a.name: ClosureItem(a, va),
a2.name: ClosureItem(a2, va)
})
self.assertEqual(expected_closure, closure)
self.assertEqual("", str(biiout))
def deserialize(data):
try:
res = HiveDependencies()
res.dep_table = BlockVersionTable.deserialize(data[HiveDependencies.SERIAL_DEP_TABLE])
res.references = References.deserialize(data[HiveDependencies.SERIAL_REFERENCES])
res.closure = Closure.deserialize(data[HiveDependencies.SERIAL_CLOSURE])
res.src_graph = BlockVersionGraph.deserialize(data[HiveDependencies.SERIAL_SRC_GRAPH])
res.dep_graph = BlockVersionGraph.deserialize(data[HiveDependencies.SERIAL_DEP_GRAPH])
return res
except Exception as e:
raise BiiSerializationException(e)
def deserialize(data):
try:
res = HiveDependencies()
res.dep_table = BlockVersionTable.deserialize(
data[HiveDependencies.SERIAL_DEP_TABLE])
res.references = References.deserialize(
data[HiveDependencies.SERIAL_REFERENCES])
res.closure = Closure.deserialize(
data[HiveDependencies.SERIAL_CLOSURE])
res.src_graph = BlockVersionGraph.deserialize(
data[HiveDependencies.SERIAL_SRC_GRAPH])
res.dep_graph = BlockVersionGraph.deserialize(
data[HiveDependencies.SERIAL_DEP_GRAPH])
return res
except Exception as e:
raise BiiSerializationException(e)
def test_add_item(self):
c1 = Closure()
r1 = Cell('user/block/name1')
version = BlockVersion(BRLBlock('owner/user/block/branch'), 13)
biiout = OutputStream()
resource = Resource(r1, None)
c1.add_item(resource, version, biiout)
r2 = Cell('user/block/name1')
r2.hasMain = True
resource2 = Resource(r2, None)
version2 = BlockVersion(BRLBlock('owner/user/block/branch'), 14)
c1.add_item(resource2, version2, biiout)
self.assertEqual(ClosureItem(resource, version), c1['user/block/name1'])
self.assertIn('Incompatible dependency', str(biiout))
class HiveDependencies(object):
""" This is a caching object for storing the block and version level dependencies of the
current graph, as well as the closure
"""
def __init__(self):
self.references = None # References()
self.closure = Closure() # {BlockCellName: Resource}
self.src_graph = BlockVersionGraph() # BlockVersionGraph
self.dep_graph = BlockVersionGraph() # BlockVersionGraph
@property
def version_graph(self):
return self.src_graph + self.dep_graph
@property
def bii_config(self):
''' Return a dict of biicode.conf objects, e.g.:
{user/superblock: <bii_config_object>,
user_2/otherblock: <bii_config_object>}
'''
return self.closure.bii_config()
def build_closure(biiapi, references, base_table, settings=None, biiout=None):
graph = BlockVersionGraph()
closure = Closure()
visited = set()
visited_block_versions = set()
overwrites = defaultdict(set)
if references:
assert not ([
r for r in references.explode()
if r.block_version.block_name in base_table
and base_table[r.block_version.block_name] != r.block_version
])
frontier = [(references, base_table)
] # Of tuple (references, base_table)
else:
frontier = []
class Visited(namedtuple('Visited', 'version cell_name, dep_table')):
def __hash__(self):
return hash((self.version, self.cell_name))
while frontier:
references, base_table = frontier.pop()
retrieved = biiapi.get_published_resources(references)
for block_version, resources in retrieved.iteritems():
graph.add_node(block_version)
dep_targets = set()
if block_version not in visited_block_versions:
cell_names = biiapi.get_cells_snapshot(block_version)
_add_implicit_targets(dep_targets, cell_names, block_version)
visited_block_versions.add(block_version)
try:
up_table = biiapi.get_dep_table(block_version)
except Exception as e:
raise BiiException(
'%s\nbiicode needs compare your local "%s" block with your last'
' version published one. If you want to delete it, delete the '
'folder in the filesystem.' %
(str(e), block_version.block_name))
effective, propagate, overwrite = compute_effective(
base_table, up_table, block_version.block_name)
if overwrite:
overwrites[block_version].update(overwrite)
for cell_name, resource in resources.iteritems():
closure.add_item(resource, block_version, biiout)
visited.add(Visited(block_version, cell_name, base_table))
_update_dep_targets(dep_targets, resource, settings, biiout)
other_refs = References()
self_refs = References()
for target in dep_targets:
if target.block_name == block_version.block_name:
next_visit = Visited(block_version, target.cell_name,
base_table)
if next_visit not in visited:
self_refs[block_version].add(target.cell_name)
else:
next_version = effective[target.block_name]
graph.add_edge(block_version, next_version)
if next_version.time is not None:
next_visit = Visited(next_version, target.cell_name,
propagate)
if next_visit not in visited:
other_refs[next_version].add(target.cell_name)
if other_refs:
frontier.append((other_refs, propagate))
if self_refs:
frontier.append((self_refs, base_table))
return graph, closure, overwrites
def __init__(self):
self.references = None # References()
self.closure = Closure() # {BlockCellName: Resource}
self.src_graph = BlockVersionGraph() # BlockVersionGraph
self.dep_graph = BlockVersionGraph() # BlockVersionGraph
def build_closure(biiapi, references, base_table, settings=None, biiout=None):
graph = BlockVersionGraph()
closure = Closure()
visited = set()
visited_block_versions = set()
overwrites = defaultdict(set)
if references:
assert not ([r for r in references.explode() if r.block_version.block_name in base_table
and base_table[r.block_version.block_name] != r.block_version])
frontier = [(references, base_table)] # Of tuple (references, base_table)
else:
frontier = []
class Visited(namedtuple('Visited', 'version cell_name, dep_table')):
def __hash__(self):
return hash((self.version, self.cell_name))
while frontier:
references, base_table = frontier.pop()
retrieved = biiapi.get_published_resources(references)
for block_version, resources in retrieved.iteritems():
graph.add_node(block_version)
dep_targets = set()
if block_version not in visited_block_versions:
cell_names = biiapi.get_cells_snapshot(block_version)
_add_implicit_targets(dep_targets, cell_names, block_version)
visited_block_versions.add(block_version)
try:
up_table = biiapi.get_dep_table(block_version)
except Exception as e:
raise BiiException('%s\nbiicode needs compare your local "%s" block with your last'
' version published one. If you want to delete it, delete the '
'folder in the filesystem.' % (str(e),
block_version.block_name))
effective, propagate, overwrite = compute_effective(base_table, up_table,
block_version.block_name)
if overwrite:
overwrites[block_version].update(overwrite)
for cell_name, resource in resources.iteritems():
closure.add_item(resource, block_version, biiout)
visited.add(Visited(block_version, cell_name, base_table))
_update_dep_targets(dep_targets, resource, settings, biiout)
other_refs = References()
self_refs = References()
for target in dep_targets:
if target.block_name == block_version.block_name:
next_visit = Visited(block_version, target.cell_name, base_table)
if next_visit not in visited:
self_refs[block_version].add(target.cell_name)
else:
next_version = effective[target.block_name]
graph.add_edge(block_version, next_version)
if next_version.time is not None:
next_visit = Visited(next_version, target.cell_name, propagate)
if next_visit not in visited:
other_refs[next_version].add(target.cell_name)
if other_refs:
frontier.append((other_refs, propagate))
if self_refs:
frontier.append((self_refs, base_table))
return graph, closure, overwrites
